Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver)!: expose typed native-window SDK flow (#3683)

Francesco Bonacci75b04aac03ed
diff --git a/.github/scripts/cua-driver-mcp-compat/expected-tools.json b/.github/scripts/cua-driver-mcp-compat/expected-tools.json
index 363114c362..6297759235 100644
--- a/.github/scripts/cua-driver-mcp-compat/expected-tools.json
+++ b/.github/scripts/cua-driver-mcp-compat/expected-tools.json
@@ -1,9 +1,9 @@
{
"serverName": "cua-driver",
"outputSchemaCountByPlatform": {
- "darwin": 32,
- "linux": 36,
- "win32": 32
+ "darwin": 34,
+ "linux": 38,
+ "win32": 34
},
"baseTools": [
"bring_to_front",
diff --git a/.github/scripts/tests/test_cua_driver_release_wiring.py b/.github/scripts/tests/test_cua_driver_release_wiring.py
index e9a21486e4..5e05886b3d 100644
--- a/.github/scripts/tests/test_cua_driver_release_wiring.py
+++ b/.github/scripts/tests/test_cua_driver_release_wiring.py
@@ -858,7 +858,7 @@ fi
self.assertEqual(len(expected["baseTools"]), 56)
self.assertEqual(
expected["outputSchemaCountByPlatform"],
- {"darwin": 32, "linux": 36, "win32": 32},
+ {"darwin": 34, "linux": 38, "win32": 34},
)
self.assertEqual(
expected["platformTools"],
diff --git a/docs/content/docs/how-to-guides/driver/use-sdk-in-process.mdx b/docs/content/docs/how-to-guides/driver/use-sdk-in-process.mdx
index 678136226f..f17f7ddb8d 100644
--- a/docs/content/docs/how-to-guides/driver/use-sdk-in-process.mdx
+++ b/docs/content/docs/how-to-guides/driver/use-sdk-in-process.mdx
@@ -93,7 +93,9 @@ try {
} finally {
await driver.endSession(EndSessionInput.new({}));
await driver.shutdown();
- driver.uniffiDestroy();
+ if ('uniffiDestroy' in driver && typeof driver.uniffiDestroy === 'function') {
+ driver.uniffiDestroy();
+ }
}
```
@@ -105,6 +107,186 @@ reuse its implicit session. End it explicitly when useful; `shutdown()` also
runs cleanup. `shutdown()` is idempotent and rejects new work after shutdown
while allowing already-admitted operations to finish.
+## Discover and act on an exact native window
+
+This typed window API targets the next breaking release. Version 0.25 supports
+native-window operations through the generic tool surface, but does not expose
+these generated methods. Upgrade the bindings and native library together.
+
+The example expects a test application named `Window Demo`, a window titled
+`Counter`, a button labeled `Increment`, and an accessible counter labeled
+`Count` whose value changes when clicked. Replace those exact identities and the
+postcondition with ones you control. It requires exactly one match among the
+returned elements and refuses degraded or truncated snapshots. Accessibility
+results can be a projection: `elementsComplete` / `elements_complete` can be
+false even for a healthy snapshot, so a unique returned label does not prove
+that no matching element exists outside the projection. Use a known control
+identity and postcondition for your application.
+
+<Tabs groupId="language" persist items={['Python', 'TypeScript']}>
+ <Tab value="Python">
+
+```python
+import asyncio
+
+from cua_driver import (
+ ActionTarget, ClickButton, ClickInput, ClickPosition, CuaDriver,
+ GetWindowStateInput, InputDeliveryMode, ListAppsInput, ListWindowsInput,
+)
+
+
+def unique(items, description):
+ if len(items) != 1:
+ raise RuntimeError(f"Expected one {description}, found {len(items)}")
+ return items[0]
+
+
+async def main() -> None:
+ driver = CuaDriver.create()
+ try:
+ apps = await driver.list_apps(ListAppsInput())
+ app = unique([a for a in apps.apps if a.name == "Window Demo" and a.running], "app")
+ windows = await driver.list_windows(
+ ListWindowsInput(pid=app.pid, on_screen_only=True)
+ )
+ window = unique(
+ [w for w in windows.windows if w.pid == app.pid and w.title == "Counter"],
+ "window",
+ )
+ target = ActionTarget.WINDOW(pid=app.pid, window_id=window.window_id)
+ capture = GetWindowStateInput(
+ pid=app.pid, window_id=window.window_id, session=None, query=None,
+ include_accessibility_tree=True, include_screenshot=True,
+ screenshot_out_file=None, max_elements=None, max_depth=None,
+ max_dimension=None,
+ )
+
+ async def snapshot():
+ state = await driver.get_window_state(capture)
+ if state.degraded or state.truncated:
+ raise RuntimeError("Window snapshot is degraded or truncated")
+ return state
+
+ before = await snapshot()
+ button = unique([e for e in (before.elements or []) if e.label == "Increment"], "button")
+ counter = unique([e for e in (before.elements or []) if e.label == "Count"], "counter")
+ if not button.element_token or counter.value is None:
+ raise RuntimeError("Snapshot lacks the button token or counter value")
+ await driver.click(ClickInput(
+ target=target,
+ position=ClickPosition.ELEMENT(element_token=button.element_token),
+ delivery_mode=InputDeliveryMode.BACKGROUND,
+ session=None, button=ClickButton.LEFT, count=1,
+ ))
+ after = await snapshot()
+ updated = unique([e for e in (after.elements or []) if e.label == "Count"], "counter")
+ if updated.value is None or updated.value == counter.value:
+ raise RuntimeError("Click returned, but the counter change was not verified")
+ print("Verified counter change:", counter.value, "->", updated.value)
+ for image in after.images:
+ print(image.mime_type, len(image.data_base64))
+ finally:
+ await driver.shutdown()
+
+
+asyncio.run(main())
+```
+
+ </Tab>
+ <Tab value="TypeScript">
+
+```ts
+import {
+ ActionTarget, ClickButton, ClickInput, ClickPosition, CuaDriver,
+ GetWindowStateInput, InputDeliveryMode, ListAppsInput, ListWindowsInput,
+} from '@trycua/cua-driver';
+
+function unique<T>(items: T[], description: string): T {
+ if (items.length !== 1) {
+ throw new Error(`Expected one ${description}, found ${items.length}`);
+ }
+ return items[0]!;
+}
+
+const driver = CuaDriver.create(undefined);
+try {
+ const apps = await driver.listApps(ListAppsInput.new({}));
+ const app = unique(apps.apps.filter(a => a.name === 'Window Demo' && a.running), 'app');
+ const windows = await driver.listWindows(
+ ListWindowsInput.new({ pid: app.pid, onScreenOnly: true })
+ );
+ const window = unique(
+ windows.windows.filter(w => w.pid === app.pid && w.title === 'Counter'), 'window'
+ );
+ // Preserve windowId as bigint; converting to number can lose precision.
+ const target = new ActionTarget.Window({ pid: app.pid, windowId: window.windowId });
+ const capture = GetWindowStateInput.new({
+ pid: app.pid, windowId: window.windowId,
+ includeAccessibilityTree: true, includeScreenshot: true,
+ });
+ async function snapshot() {
+ const state = await driver.getWindowState(capture);
+ if (state.degraded || state.truncated) {
+ throw new Error('Window snapshot is degraded or truncated');
+ }
+ return state;
+ }
+ const before = await snapshot();
+ const button = unique((before.elements ?? []).filter(e => e.label === 'Increment'), 'button');
+ const counter = unique((before.elements ?? []).filter(e => e.label === 'Count'), 'counter');
+ if (!button.elementToken || counter.value === undefined) {
+ throw new Error('Snapshot lacks the button token or counter value');
+ }
+ await driver.click(ClickInput.new({
+ target,
+ position: new ClickPosition.Element({ elementToken: button.elementToken }),
+ deliveryMode: InputDeliveryMode.Background,
+ button: ClickButton.Left, count: 1,
+ }));
+ const after = await snapshot();
+ const updated = unique((after.elements ?? []).filter(e => e.label === 'Count'), 'counter');
+ if (updated.value === undefined || updated.value === counter.value) {
+ throw new Error('Click returned, but the counter change was not verified');
+ }
+ console.log('Verified counter change:', counter.value, '->', updated.value);
+ for (const image of after.images) {
+ console.log(image.mimeType, image.dataBase64.length);
+ }
+} finally {
+ try {
+ await driver.shutdown();
+ } finally {
+ if ('uniffiDestroy' in driver && typeof driver.uniffiDestroy === 'function') {
+ driver.uniffiDestroy();
+ }
+ }
+}
+```
+
+ </Tab>
+</Tabs>
+
+`click` returns `ActionResult` directly and raises `DriverError.Tool` on refusal.
+It does not return a `ToolResult` envelope. Other action methods retain their
+existing return types. A returned action result does not prove the intended UI
+change; the second snapshot checks that separately. For asynchronous UIs, poll
+fresh snapshots with a bounded deadline for your postcondition without repeating
+the click.
+
+Background delivery never authorizes an automatic foreground retry. Handle
+unsupported OS or application routes explicitly. If a token is stale, refresh
+the same window and resolve the element again. Do not reuse tokens after a
+snapshot, window, or session change.
+
+Inspect `snapshotId` / `snapshot_id`, degradation fields, and screenshot metadata
+when using a window capture. Window screenshots and desktop screenshots use
+different coordinate spaces. A window target takes coordinates from its fresh
+window screenshot; do not add the screen origin yourself. Inspect dimensions,
+scale, and frame validity, and account for any image resizing your application
+performs. The token example avoids pixel-coordinate conversion.
+See the [native-window migration guide](https://github.com/trycua/cua/blob/2c4f19d2aa455b6ae2f79d2bf982e9b3c2356fda/libs/cua-driver/docs/native-window-sdk-migration.md)
+for the complete compatibility changes.
+
If an external agent must also connect to your signed desktop application, use
[Expose MCP from a desktop app](/how-to-guides/driver/expose-mcp-from-desktop-app)
instead.
diff --git a/libs/cua-driver/compat-fixtures/README.md b/libs/cua-driver/compat-fixtures/README.md
index 136773ccc6..02038b94bf 100644
--- a/libs/cua-driver/compat-fixtures/README.md
+++ b/libs/cua-driver/compat-fixtures/README.md
@@ -26,6 +26,15 @@ Additive manifest fields, tools, and package metadata are permitted. Removing
or changing an item recorded here requires an explicit compatibility decision
and an intentional fixture update.
+RFC 3682 accepts a breaking native SDK click change: `ClickInput` now requires
+an `ActionTarget`, a `ClickPosition` (coordinates or an element token), and an
+explicit background/foreground `InputDeliveryMode`. `click` returns
+`ActionResult` directly and raises `DriverError.Tool` for tool refusals. The
+Python click signature is updated accordingly; the TypeScript baseline did not
+freeze that method. Current language tests cover the new signatures and typed
+native-window discovery and observation. All other frozen package signatures,
+the CLI/MCP fixtures, and the baseline application fixtures remain unchanged.
+
RFC 2549 independently accepts one additive CLI change: `cua-driver mcp
--direct`. Bare MCP behavior remains platform-defined (direct on Windows and
Linux, signed app service on macOS), while `--socket` continues to select an
diff --git a/libs/cua-driver/compat-fixtures/python-package.json b/libs/cua-driver/compat-fixtures/python-package.json
index 58a81d74e0..9446f29b08 100644
--- a/libs/cua-driver/compat-fixtures/python-package.json
+++ b/libs/cua-driver/compat-fixtures/python-package.json
@@ -63,7 +63,7 @@
"create(cls, options: typing.Optional[DriverOptions]) -> CuaDriver",
"create_with_client_kind(cls, options: typing.Optional[DriverOptions], client_kind: SdkClientKind) -> CuaDriver",
"async call_tool(self, name: str, arguments_json: str) -> ToolResult",
- "async click(self, input: cua_driver._native_contract.ClickInput) -> ToolResult",
+ "async click(self, input: cua_driver._native_contract.ClickInput) -> cua_driver._native_contract.ActionResult",
"async drag(self, input: cua_driver._native_contract.DragInput) -> ToolResult",
"async end_session(self, input: cua_driver._native_contract.EndSessionInput) -> cua_driver._native_contract.EndSessionOutput",
"async escalate_session(self, input: cua_driver._native_contract.EscalateSessionInput) -> cua_driver._native_contract.SessionStateOutput",
diff --git a/libs/cua-driver/contract/README.md b/libs/cua-driver/contract/README.md
index e3e37b319f..2281ec9187 100644
--- a/libs/cua-driver/contract/README.md
+++ b/libs/cua-driver/contract/README.md
@@ -1,16 +1,16 @@
# Experimental cua-driver SDK contract
-This directory contains the checked-in, generated contract for the first
-portable cua-driver SDK slice. The Rust crate at
+This directory contains the checked-in, generated portable cua-driver SDK
+contract. The Rust crate at
`rust/crates/cua-driver-contract` is the source of truth. It generates this
manifest and exports the request/result records consumed by the live daemon and
the UniFFI SDK. Python and TypeScript bindings are generated from the compiled
Rust library by `scripts/generate-uniffi-bindings.mjs`.
-The prototype intentionally keeps execution, platform integration, policy, and
-permission handling in the native Cua Driver process. Imported SDKs call the
-daemon through the shared Rust socket client; they do not route through MCP or
-embed platform code. Agents independently use the public `cua-driver mcp`
+Execution, platform integration, policy, and permission handling remain in
+Rust. `CuaDriver.create()` owns that runtime in the importing process;
+`connect()` uses an existing daemon through the shared Rust socket client.
+Agents independently use the public `cua-driver mcp`
surface through their runtime's existing MCP client.
## Scope and compatibility
@@ -42,6 +42,15 @@ It also covers the portable whole-desktop loop:
- `clipboard_read` for available types and opt-in plain-text readback
- `clipboard_write` for text, image, and file-URL clipboard content
+The typed native-window flow includes `list_apps`, `list_windows`, and
+`get_window_state`, with discovery records, snapshot-bound element tokens,
+optional accessibility metadata, and screenshot images attached to the typed
+snapshot. `click` accepts one `ClickPosition` (coordinates or element token), an
+explicit `ActionTarget`, and an explicit `InputDeliveryMode`, and returns
+`ActionResult` directly. Native refusals become `DriverError.Tool`.
+See the [0.8 SDK contract migration](../docs/native-window-sdk-migration.md)
+for the intentional SDK break and unchanged CLI/MCP wire forms.
+
The canonical session-owned cursor slice is shared exactly by MCP and both
generated SDKs:
@@ -74,8 +83,9 @@ target is a tagged union: `{kind:"window", pid, window_id}` or
be mixed with `target`. Desktop contracts
are marked `portable_subset`: their typed Rust inputs are a deliberately
narrower projection of the richer macOS, Linux, and Windows runtime schemas.
-Each platform's desktop branch deserializes that projection before acting,
-while window/element-only fields remain in the richer live schema. Successful
+Platform handlers retain their existing wire parsers, including the legacy
+desktop-coordinate click projection. The SDK serializes its stricter click
+addressing to those same native fields. Successful
SDK-path structured payloads are validated against the shared Rust output
types in the live registry.
@@ -99,14 +109,14 @@ Compatibility is tracked separately at each boundary:
| Field | Current | Meaning |
| --- | --- | --- |
-| `contract_version` | `0.7.0` | Generated manifest and typed SDK shape |
+| `contract_version` | `0.8.0` | Generated manifest and typed SDK shape |
| `tools_list_schema_version` | `1` | cua-driver `tools/list` extension shape |
| `capability_version` | `1` | Additive capability-token vocabulary |
| `mcp_protocol_version` | `2025-06-18` | MCP initialization protocol served to agent runtimes |
-This implementation does not use WASM. UniFFI distributes one Rust
-daemon-client implementation to Python and Node while preserving the daemon's
-permission identity and runtime ownership. The language packages do not
+This implementation does not use WASM. UniFFI distributes the shared Rust
+implementation to Python and Node. Permission identity follows the selected
+runtime host. The language packages do not
generate or maintain separate MCP transports.
## Generate and verify
diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.json
index 147a07eab7..7245868e34 100644
--- a/libs/cua-driver/contract/manifest.json
+++ b/libs/cua-driver/contract/manifest.json
@@ -1,7 +1,7 @@
{
"generated_notice": "Generated by cua-contract-gen; do not edit by hand.",
"experimental": true,
- "contract_version": "0.7.0",
+ "contract_version": "0.8.0",
"tools_list_schema_version": "1",
"capability_version": "1",
"mcp_protocol_version": "2025-06-18",
@@ -9,7 +9,7 @@
"tools": [
{
"name": "click",
- "description": "Click an absolute point in get_desktop_state coordinates without targeting a window.",
+ "description": "Click coordinates or a snapshot-bound element with an explicit target and delivery mode.",
"platforms": [
"macos",
"windows",
@@ -18,7 +18,8 @@
"capabilities": [
"input.pointer.click",
"input.pointer.click.left",
- "accessibility.element_tokens"
+ "accessibility.element_tokens",
+ "input.delivery_mode"
],
"annotations": {
"read_only": false,
@@ -32,8 +33,41 @@
},
"input_schema": {
"additionalProperties": false,
+ "oneOf": [
+ {
+ "not": {
+ "required": [
+ "element_token"
+ ]
+ },
+ "required": [
+ "x",
+ "y"
+ ]
+ },
+ {
+ "not": {
+ "anyOf": [
+ {
+ "required": [
+ "x"
+ ]
+ },
+ {
+ "required": [
+ "y"
+ ]
+ }
+ ]
+ },
+ "required": [
+ "element_token"
+ ]
+ }
+ ],
"properties": {
"button": {
+ "default": null,
"enum": [
"left",
"right",
@@ -42,82 +76,86 @@
"type": "string"
},
"count": {
+ "default": null,
"maximum": 3,
"minimum": 1,
"type": "integer"
},
- "scope": {
- "const": "desktop",
- "description": "Deprecated flat desktop target retained for wire compatibility."
+ "delivery_mode": {
+ "enum": [
+ "background",
+ "foreground"
+ ],
+ "type": "string"
+ },
+ "element_token": {
+ "default": null,
+ "type": "string"
},
"session": {
+ "default": null,
"description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
"type": "string"
},
"target": {
- "anyOf": [
+ "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
+ "oneOf": [
{
- "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
- "oneOf": [
- {
- "additionalProperties": true,
- "properties": {
- "kind": {
- "const": "window",
- "type": "string"
- },
- "pid": {
- "format": "uint32",
- "minimum": 0,
- "type": "integer"
- },
- "window_id": {
- "format": "uint64",
- "minimum": 0,
- "type": "integer"
- }
- },
- "required": [
- "kind",
- "pid",
- "window_id"
- ],
- "type": "object"
+ "additionalProperties": true,
+ "properties": {
+ "kind": {
+ "const": "window",
+ "type": "string"
},
- {
- "additionalProperties": true,
- "properties": {
- "display_id": {
- "type": "string"
- },
- "kind": {
- "const": "desktop",
- "type": "string"
- }
- },
- "required": [
- "kind",
- "display_id"
- ],
- "type": "object"
+ "pid": {
+ "format": "uint32",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "window_id": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": "integer"
}
- ]
+ },
+ "required": [
+ "kind",
+ "pid",
+ "window_id"
+ ],
+ "type": "object"
},
{
- "type": "null"
+ "additionalProperties": true,
+ "properties": {
+ "display_id": {
+ "type": "string"
+ },
+ "kind": {
+ "const": "desktop",
+ "type": "string"
+ }
+ },
+ "required": [
+ "kind",
+ "display_id"
+ ],
+ "type": "object"
}
]
},
"x": {
+ "default": null,
"type": "number"
},
"y": {
+ "default": null,
"type": "number"
}
},
"required": [
- "x",
- "y"
+ "target",
+ "delivery_mode"
],
"type": "object"
},
@@ -1424,181 +1462,564 @@
}
},
{
- "name": "hotkey",
- "description": "Press a key chord in the foreground desktop application.",
+ "name": "get_window_state",
+ "description": "Observe an exact native window with snapshot-bound elements and optional screenshots.",
"platforms": [
"macos",
"windows",
"linux"
],
"capabilities": [
- "input.keyboard.hotkey"
+ "accessibility.window_state",
+ "accessibility.tree",
+ "accessibility.tree.structured",
+ "accessibility.tree.bounded",
+ "accessibility.element_tokens",
+ "screen.capture",
+ "screen.capture.window"
],
"annotations": {
- "read_only": false,
- "destructive": true,
+ "read_only": true,
+ "destructive": false,
"idempotent": false,
- "open_world": true
+ "open_world": false
},
"schema_mode": "portable_subset",
"cursor_semantics": {
- "action": "key"
+ "action": "observe"
},
"input_schema": {
"additionalProperties": false,
"properties": {
- "keys": {
- "items": {
- "type": "string"
- },
- "minItems": 2,
- "type": "array"
+ "include_accessibility_tree": {
+ "type": "boolean"
},
- "scope": {
- "const": "desktop",
- "description": "Deprecated flat desktop target retained for wire compatibility."
+ "include_screenshot": {
+ "type": "boolean"
+ },
+ "max_depth": {
+ "minimum": 1,
+ "type": "integer"
+ },
+ "max_dimension": {
+ "minimum": 1,
+ "type": "integer"
+ },
+ "max_elements": {
+ "minimum": 1,
+ "type": "integer"
+ },
+ "pid": {
+ "maximum": 4294967295,
+ "minimum": 1,
+ "type": "integer"
+ },
+ "query": {
+ "type": "string"
+ },
+ "screenshot_out_file": {
+ "type": "string"
},
"session": {
- "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
"type": "string"
},
- "target": {
- "anyOf": [
- {
- "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
- "oneOf": [
- {
- "additionalProperties": true,
- "properties": {
- "kind": {
- "const": "window",
- "type": "string"
- },
- "pid": {
- "format": "uint32",
- "minimum": 0,
- "type": "integer"
- },
- "window_id": {
- "format": "uint64",
- "minimum": 0,
- "type": "integer"
- }
- },
- "required": [
- "kind",
- "pid",
- "window_id"
- ],
- "type": "object"
- },
- {
- "additionalProperties": true,
- "properties": {
- "display_id": {
- "type": "string"
- },
- "kind": {
- "const": "desktop",
- "type": "string"
- }
- },
- "required": [
- "kind",
- "display_id"
- ],
- "type": "object"
- }
- ]
- },
- {
- "type": "null"
- }
- ]
+ "window_id": {
+ "minimum": 1,
+ "type": "integer"
}
},
"required": [
- "keys"
+ "pid",
+ "window_id"
],
"type": "object"
},
"success_output_schema": {
- "additionalProperties": false,
+ "additionalProperties": true,
"properties": {
- "delivery": {
- "additionalProperties": false,
- "properties": {
- "delivered_count": {
- "format": "uint32",
- "minimum": 0,
- "type": [
- "integer",
- "null"
- ]
- },
- "mode": {
- "enum": [
- "background",
- "foreground",
- "not_applicable",
- "unknown"
- ],
- "type": "string"
- }
- },
- "required": [
- "mode"
- ],
+ "app_name": {
"type": [
- "object",
+ "string",
"null"
]
},
- "effect": {
- "enum": [
- "confirmed",
- "partial",
- "unverifiable",
- "suspected_noop",
- "refused"
- ],
- "type": "string"
+ "degraded": {
+ "type": [
+ "boolean",
+ "null"
+ ]
},
- "escalation": {
- "additionalProperties": false,
- "properties": {
- "reason": {
- "enum": [
- "route_unavailable",
- "delivery_failed",
- "effect_unconfirmed",
- "suspected_noop",
- "permission_required"
- ],
- "type": "string"
- },
- "target": {
- "enum": [
- "pixel",
- "foreground",
- "page",
- "session"
- ],
- "type": "string"
- }
- },
- "required": [
- "target",
- "reason"
- ],
+ "degraded_reason": {
"type": [
- "object",
+ "string",
"null"
]
},
- "evidence": {
- "items": {
- "additionalProperties": false,
+ "element_count": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "elements": {
+ "items": {
+ "properties": {
+ "actions": {
+ "items": {
+ "type": "string"
+ },
+ "type": [
+ "array",
+ "null"
+ ]
+ },
+ "depth": {
+ "format": "uint32",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "element_index": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "element_token": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "enabled": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "frame": {
+ "properties": {
+ "h": {
+ "format": "double",
+ "type": "number"
+ },
+ "w": {
+ "format": "double",
+ "type": "number"
+ },
+ "x": {
+ "format": "double",
+ "type": "number"
+ },
+ "y": {
+ "format": "double",
+ "type": "number"
+ }
+ },
+ "required": [
+ "x",
+ "y",
+ "w",
+ "h"
+ ],
+ "type": [
+ "object",
+ "null"
+ ]
+ },
+ "in_web_content": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "max": {
+ "format": "double",
+ "type": [
+ "number",
+ "null"
+ ]
+ },
+ "min": {
+ "format": "double",
+ "type": [
+ "number",
+ "null"
+ ]
+ },
+ "parent_index": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "role": {
+ "type": "string"
+ },
+ "selected": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "value": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "value_description": {
+ "type": [
+ "string",
+ "null"
+ ]
+ }
+ },
+ "required": [
+ "element_index",
+ "role",
+ "depth"
+ ],
+ "type": "object"
+ },
+ "type": [
+ "array",
+ "null"
+ ]
+ },
+ "elements_complete": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "filtered_element_count": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "pid": {
+ "format": "uint32",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "returned_element_count": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "screenshot_file_path": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "screenshot_frame_valid": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "screenshot_height": {
+ "format": "uint32",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "screenshot_mime_type": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "screenshot_scale": {
+ "format": "double",
+ "type": [
+ "number",
+ "null"
+ ]
+ },
+ "screenshot_width": {
+ "format": "uint32",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "snapshot_id": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "total_element_count": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "tree_markdown": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "truncated": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "truncation_reason": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "window_bounds": {
+ "properties": {
+ "height": {
+ "format": "double",
+ "type": "number"
+ },
+ "width": {
+ "format": "double",
+ "type": "number"
+ },
+ "x": {
+ "format": "double",
+ "type": "number"
+ },
+ "y": {
+ "format": "double",
+ "type": "number"
+ }
+ },
+ "required": [
+ "x",
+ "y",
+ "width",
+ "height"
+ ],
+ "type": [
+ "object",
+ "null"
+ ]
+ },
+ "window_id": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "window_title": {
+ "type": [
+ "string",
+ "null"
+ ]
+ }
+ },
+ "required": [
+ "pid",
+ "window_id"
+ ],
+ "type": "object"
+ }
+ },
+ {
+ "name": "hotkey",
+ "description": "Press a key chord in the foreground desktop application.",
+ "platforms": [
+ "macos",
+ "windows",
+ "linux"
+ ],
+ "capabilities": [
+ "input.keyboard.hotkey"
+ ],
+ "annotations": {
+ "read_only": false,
+ "destructive": true,
+ "idempotent": false,
+ "open_world": true
+ },
+ "schema_mode": "portable_subset",
+ "cursor_semantics": {
+ "action": "key"
+ },
+ "input_schema": {
+ "additionalProperties": false,
+ "properties": {
+ "keys": {
+ "items": {
+ "type": "string"
+ },
+ "minItems": 2,
+ "type": "array"
+ },
+ "scope": {
+ "const": "desktop",
+ "description": "Deprecated flat desktop target retained for wire compatibility."
+ },
+ "session": {
+ "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+ "type": "string"
+ },
+ "target": {
+ "anyOf": [
+ {
+ "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
+ "oneOf": [
+ {
+ "additionalProperties": true,
+ "properties": {
+ "kind": {
+ "const": "window",
+ "type": "string"
+ },
+ "pid": {
+ "format": "uint32",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "window_id": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": "integer"
+ }
+ },
+ "required": [
+ "kind",
+ "pid",
+ "window_id"
+ ],
+ "type": "object"
+ },
+ {
+ "additionalProperties": true,
+ "properties": {
+ "display_id": {
+ "type": "string"
+ },
+ "kind": {
+ "const": "desktop",
+ "type": "string"
+ }
+ },
+ "required": [
+ "kind",
+ "display_id"
+ ],
+ "type": "object"
+ }
+ ]
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ },
+ "required": [
+ "keys"
+ ],
+ "type": "object"
+ },
+ "success_output_schema": {
+ "additionalProperties": false,
+ "properties": {
+ "delivery": {
+ "additionalProperties": false,
+ "properties": {
+ "delivered_count": {
+ "format": "uint32",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "mode": {
+ "enum": [
+ "background",
+ "foreground",
+ "not_applicable",
+ "unknown"
+ ],
+ "type": "string"
+ }
+ },
+ "required": [
+ "mode"
+ ],
+ "type": [
+ "object",
+ "null"
+ ]
+ },
+ "effect": {
+ "enum": [
+ "confirmed",
+ "partial",
+ "unverifiable",
+ "suspected_noop",
+ "refused"
+ ],
+ "type": "string"
+ },
+ "escalation": {
+ "additionalProperties": false,
+ "properties": {
+ "reason": {
+ "enum": [
+ "route_unavailable",
+ "delivery_failed",
+ "effect_unconfirmed",
+ "suspected_noop",
+ "permission_required"
+ ],
+ "type": "string"
+ },
+ "target": {
+ "enum": [
+ "pixel",
+ "foreground",
+ "page",
+ "session"
+ ],
+ "type": "string"
+ }
+ },
+ "required": [
+ "target",
+ "reason"
+ ],
+ "type": [
+ "object",
+ "null"
+ ]
+ },
+ "evidence": {
+ "items": {
+ "additionalProperties": false,
"properties": {
"kind": {
"enum": [
@@ -1808,6 +2229,95 @@
"type": "object"
}
},
+ {
+ "name": "list_apps",
+ "description": "Discover installed and running native applications.",
+ "platforms": [
+ "macos",
+ "windows",
+ "linux"
+ ],
+ "capabilities": [
+ "app.list"
+ ],
+ "annotations": {
+ "read_only": true,
+ "destructive": false,
+ "idempotent": true,
+ "open_world": false
+ },
+ "schema_mode": "portable_subset",
+ "cursor_semantics": {
+ "action": "observe"
+ },
+ "input_schema": {
+ "additionalProperties": false,
+ "properties": {},
+ "required": [],
+ "type": "object"
+ },
+ "success_output_schema": {
+ "additionalProperties": true,
+ "properties": {
+ "apps": {
+ "items": {
+ "properties": {
+ "active": {
+ "type": "boolean"
+ },
+ "bundle_id": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "kind": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "last_used": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "launch_path": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "name": {
+ "type": "string"
+ },
+ "pid": {
+ "format": "uint32",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "running": {
+ "type": "boolean"
+ }
+ },
+ "required": [
+ "pid",
+ "name",
+ "running",
+ "active"
+ ],
+ "type": "object"
+ },
+ "type": "array"
+ }
+ },
+ "required": [
+ "apps"
+ ],
+ "type": "object"
+ }
+ },
{
"name": "list_sessions",
"description": "List content-free lifecycle summaries attached to this authenticated transport lease. It does not enumerate other callers' sessions.",
@@ -1944,6 +2454,168 @@
"type": "object"
}
},
+ {
+ "name": "list_windows",
+ "description": "Discover exact native windows and observable bounds and stacking order.",
+ "platforms": [
+ "macos",
+ "windows",
+ "linux"
+ ],
+ "capabilities": [
+ "window.list"
+ ],
+ "annotations": {
+ "read_only": true,
+ "destructive": false,
+ "idempotent": true,
+ "open_world": false
+ },
+ "schema_mode": "portable_subset",
+ "cursor_semantics": {
+ "action": "observe"
+ },
+ "input_schema": {
+ "additionalProperties": false,
+ "properties": {
+ "on_screen_only": {
+ "type": "boolean"
+ },
+ "pid": {
+ "maximum": 4294967295,
+ "minimum": 1,
+ "type": "integer"
+ }
+ },
+ "required": [],
+ "type": "object"
+ },
+ "success_output_schema": {
+ "additionalProperties": true,
+ "properties": {
+ "current_space_id": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "windows": {
+ "items": {
+ "properties": {
+ "app_name": {
+ "type": "string"
+ },
+ "bounds": {
+ "properties": {
+ "height": {
+ "format": "double",
+ "type": "number"
+ },
+ "width": {
+ "format": "double",
+ "type": "number"
+ },
+ "x": {
+ "format": "double",
+ "type": "number"
+ },
+ "y": {
+ "format": "double",
+ "type": "number"
+ }
+ },
+ "required": [
+ "x",
+ "y",
+ "width",
+ "height"
+ ],
+ "type": "object"
+ },
+ "current_space_id": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "is_on_screen": {
+ "type": "boolean"
+ },
+ "layer": {
+ "format": "int32",
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "minimized": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "on_current_space": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "pid": {
+ "maximum": 4294967295,
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "space_ids": {
+ "items": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "type": [
+ "array",
+ "null"
+ ]
+ },
+ "window_id": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "z_index": {
+ "description": "Higher values are closer to the front. Null means the provider cannot observe stacking order; callers must not infer an order from array position or treat null as zero.",
+ "type": [
+ "integer",
+ "null"
+ ]
+ }
+ },
+ "required": [
+ "window_id",
+ "pid",
+ "app_name",
+ "title",
+ "bounds",
+ "is_on_screen",
+ "z_index"
+ ],
+ "type": "object"
+ },
+ "type": "array"
+ }
+ },
+ "required": [
+ "windows"
+ ],
+ "type": "object"
+ }
+ },
{
"name": "move_cursor",
"description": "Move the real OS pointer in get_desktop_state coordinates.",
diff --git a/libs/cua-driver/docs/native-window-sdk-migration.md b/libs/cua-driver/docs/native-window-sdk-migration.md
new file mode 100644
index 0000000000..eccb4f025a
--- /dev/null
+++ b/libs/cua-driver/docs/native-window-sdk-migration.md
@@ -0,0 +1,84 @@
+# Migrate to the typed native-window SDK
+
+The next breaking Cua Driver release adds typed app discovery, window discovery,
+and native-window snapshots, and changes the SDK `click` input and return value.
+These examples target that release. Version 0.25 already supports native-window
+operations through the generic tool surface; it does not provide this typed SDK
+contract. Upgrade the generated bindings and native library together.
+
+## Replace generic window calls
+
+Use the generated methods and records directly:
+
+| Operation | TypeScript | Python | Result |
+| --- | --- | --- | --- |
+| Discover running apps | `listApps(ListAppsInput.new({}))` | `list_apps(ListAppsInput())` | `ListAppsOutput.apps` |
+| Discover an app's windows | `listWindows(ListWindowsInput.new({ pid }))` | `list_windows(ListWindowsInput(pid=pid, on_screen_only=None))` | `ListWindowsOutput.windows` |
+| Observe an exact window | `getWindowState(GetWindowStateInput.new({ pid, windowId }))` | `get_window_state(GetWindowStateInput(...))` | `WindowStateOutput` |
+
+Python's generated input constructors require explicit `None` for omitted
+optional fields. The complete example linked below supplies every window-state
+input field.
+
+Select a unique app and window using identities your application knows. An app
+may own several windows, and ordering is not a selection contract. Do not take
+the first result or substitute the active window when the intended window is
+missing. TypeScript window IDs are `bigint`; preserve them without conversion to
+`number`.
+
+`WindowStateOutput.elements` contains structured accessibility elements when
+available; screenshot-only captures can omit it. Use the snapshot's element
+tokens for actions, and inspect its optional snapshot ID, screenshot metadata,
+and degradation fields before relying on a capture.
+`elementsComplete` / `elements_complete` can be false for a healthy actionable
+projection. A missing element does not prove absence, and a unique returned
+label establishes uniqueness only among the returned elements. Use control
+identities and postconditions that your application knows.
+`images` carries base64 image data and MIME types directly. You no longer need
+to parse `ToolResult.structuredJson` / `structured_json` for these three methods.
+Other SDK methods retain their existing return types.
+
+## Replace click coordinates and handle the direct result
+
+`ClickInput` requires an exact `ActionTarget`, a `ClickPosition`, and an explicit
+`InputDeliveryMode`. Replace the old top-level `x` and `y` fields with
+`ClickPosition.Coordinates` in TypeScript or `ClickPosition.COORDINATES` in Python.
+For an accessibility element, use `ClickPosition.Element` or
+`ClickPosition.ELEMENT` with its snapshot token. Set `button`, `count`, and
+`session` when needed.
+
+`click` returns `ActionResult` directly. Remove `result.isError` / `is_error`,
+`result.text`, and JSON-envelope parsing from click-only code. A refused click
+raises `DriverError.Tool`; it is not a successful `ToolResult` carrying an error
+flag. Mixed wrappers must distinguish `ActionResult` from the `ToolResult`
+returned by methods such as `typeText` / `type_text` and `pressKey` / `press_key`.
+
+An action result describes delivery, not proof that the UI reached the intended
+state. Capture a fresh window snapshot and verify an application-specific
+postcondition. After a timeout, inspect the UI before retrying: the action may
+have landed even when the caller did not receive a result.
+
+## Preserve targeting and delivery semantics
+
+- Request background delivery explicitly. If the OS or application cannot
+ deliver that action in the background, handle the refusal. Do not retry with
+ foreground delivery automatically; that changes focus and needs a deliberate
+ caller decision.
+- Treat element tokens as snapshot-bound. If a token is stale, capture the same
+ exact window again and resolve a unique element from the fresh snapshot.
+ Never rewrite or reuse the stale token for another window or session.
+- Check platform limitations and degradation metadata. macOS, Windows, X11,
+ and Wayland do not offer identical accessibility, capture, or background
+ input routes. An unavailable route is not permission to choose another target.
+- Keep coordinate spaces distinct. Window screenshot coordinates belong to
+ that exact window capture; desktop screenshot coordinates belong to the
+ selected display. For a window target, use coordinates from its fresh window
+ screenshot, not screen-absolute coordinates; the driver performs the window
+ translation. Inspect screenshot dimensions, scale, and frame validity, and
+ do not reuse coordinates after window geometry changes or manually rescale
+ an image without accounting for that transformation.
+
+See [Use Cua Driver in process](https://cua.ai/docs/how-to-guides/driver/use-sdk-in-process)
+for complete Python and TypeScript discovery, token-click, verification, and
+shutdown examples. The [agent SDK adapters](../examples/agent-sdks/) show how to
+retain desktop coordinate tools while migrating the click result handling.
diff --git a/libs/cua-driver/examples/agent-sdks/native-tools.ts b/libs/cua-driver/examples/agent-sdks/native-tools.ts
index 3534beed63..f55aa9ecc4 100644
--- a/libs/cua-driver/examples/agent-sdks/native-tools.ts
+++ b/libs/cua-driver/examples/agent-sdks/native-tools.ts
@@ -2,10 +2,13 @@
import {
ActionTarget,
+ ActionResult,
ClickButton,
ClickInput,
+ ClickPosition,
CuaDriver,
GetDesktopStateInput,
+ InputDeliveryMode,
PressKeyInput,
ToolResult,
TypeTextInput,
@@ -32,7 +35,9 @@ export class NativeDesktopTools {
} finally {
// Generated UniFFI bindings expose deterministic handle release
// separately from asynchronous runtime shutdown.
- (this.driver as unknown as { uniffiDestroy(): void }).uniffiDestroy();
+ if ('uniffiDestroy' in this.driver && typeof this.driver.uniffiDestroy === 'function') {
+ this.driver.uniffiDestroy();
+ }
}
}
@@ -48,9 +53,9 @@ export class NativeDesktopTools {
return await this.mutateThenObserve(() =>
this.driver.click(
ClickInput.new({
- x,
- y,
+ position: new ClickPosition.Coordinates({ x, y }),
target: this.desktopTarget,
+ deliveryMode: InputDeliveryMode.Foreground,
button: ClickButton.Left,
count: 1,
})
@@ -80,11 +85,13 @@ export class NativeDesktopTools {
);
}
- private async mutateThenObserve(operation: () => Promise<ToolResult>): Promise<NativeToolResult> {
+ private async mutateThenObserve(
+ operation: () => Promise<ToolResult | ActionResult>
+ ): Promise<NativeToolResult> {
let unknownDetail: string | undefined;
try {
const result = await this.bounded(operation(), 'desktop action');
- if (result.isError) {
+ if ('isError' in result && result.isError) {
unknownDetail =
`Action reported an error and its outcome may be unknown (${result.text}). ` +
'A fresh observation follows. Do not retry until the observation proves ' +
diff --git a/libs/cua-driver/examples/agent-sdks/native_tools.py b/libs/cua-driver/examples/agent-sdks/native_tools.py
index 78a071c82b..ba184c245a 100644
--- a/libs/cua-driver/examples/agent-sdks/native_tools.py
+++ b/libs/cua-driver/examples/agent-sdks/native_tools.py
@@ -8,10 +8,13 @@ from typing import TypeVar
from cua_driver import (
ActionTarget,
+ ActionResult,
ClickButton,
ClickInput,
+ ClickPosition,
CuaDriver,
GetDesktopStateInput,
+ InputDeliveryMode,
PressKeyInput,
ToolResult,
TypeTextInput,
@@ -46,10 +49,9 @@ class NativeDesktopTools:
return await self._mutate_then_observe(
lambda: self.driver.click(
ClickInput(
- x=x,
- y=y,
+ position=ClickPosition.COORDINATES(x=x, y=y),
target=ActionTarget.DESKTOP(display_id="primary"),
- scope=None,
+ delivery_mode=InputDeliveryMode.FOREGROUND,
session=None,
button=ClickButton.LEFT,
count=1,
@@ -84,12 +86,12 @@ class NativeDesktopTools:
async def _mutate_then_observe(
self,
- operation: Callable[[], Awaitable[ToolResult]],
+ operation: Callable[[], Awaitable[ToolResult | ActionResult]],
) -> dict[str, object]:
unknown_detail: str | None = None
try:
result = await self._bounded(operation())
- if result.is_error:
+ if isinstance(result, ToolResult) and result.is_error:
unknown_detail = (
f"Action reported an error and its outcome may be unknown "
f"({result.text}). A fresh observation follows. Do not retry "
diff --git a/libs/cua-driver/python/README.md b/libs/cua-driver/python/README.md
index 23e098c893..613c9b81a2 100644
--- a/libs/cua-driver/python/README.md
+++ b/libs/cua-driver/python/README.md
@@ -71,7 +71,7 @@ async def main() -> None:
asyncio.run(main())
```
-SDK operations are asynchronous. Desktop calls return a typed `ToolResult` with
+SDK operations are asynchronous. Desktop observations return a typed `ToolResult` with
text, images, verification/error metadata, and `structured_json` / `raw_json`
for platform-extensible results. Session lifecycle calls return dedicated
generated records.
@@ -89,6 +89,28 @@ and installed with the local CLI; SDK and MCP tools select only an installed
theme ID. The built-in cursor shows the sanitized public session name in a
badge below the pointer.
+## Typed native-window migration
+
+The next breaking release adds typed app and window discovery, window snapshots,
+and token-based clicks. Version 0.25 supports native-window operations through
+the generic tool surface; it does not expose this typed window API. Upgrade the
+bindings and native library together.
+
+`list_apps` returns `ListAppsOutput`, `list_windows` returns `ListWindowsOutput`,
+and `get_window_state` returns `WindowStateOutput`. `click` takes a required exact
+target, a coordinate or element-token position, and an explicit delivery mode.
+It returns `ActionResult` directly and raises `DriverError.Tool` on refusal.
+Other action methods retain `ToolResult`.
+
+Select a unique app and window, resolve an element from a fresh snapshot, request
+background delivery explicitly, then capture again to verify the intended UI
+change. An unsupported background route must not trigger an automatic foreground
+retry. Refresh stale tokens from the same exact window.
+
+See the [migration guide](../docs/native-window-sdk-migration.md) for input and
+return-type changes, and the [complete Python and TypeScript examples](https://cua.ai/docs/how-to-guides/driver/use-sdk-in-process)
+for discovery, token selection, verification, and shutdown.
+
## Authorization integrations
`standard` is promptless for normal automation. An application that needs to
diff --git a/libs/cua-driver/python/src/cua_driver/__init__.py b/libs/cua-driver/python/src/cua_driver/__init__.py
index cfb38dc16d..22da6cd0ad 100644
--- a/libs/cua-driver/python/src/cua_driver/__init__.py
+++ b/libs/cua-driver/python/src/cua_driver/__init__.py
@@ -58,6 +58,21 @@ from ._native import (
request_mac_os_permissions,
)
from ._native_contract import (
+ AppInfo,
+ ClickPosition,
+ ElementFrame,
+ GetWindowStateInput,
+ InputDeliveryMode,
+ ListAppsInput,
+ ListAppsOutput,
+ ListWindowsInput,
+ ListWindowsOutput,
+ SnapshotImage,
+ WindowBounds,
+ WindowElement,
+ WindowInfo,
+ WindowStateOutput,
+
ActionDelivery,
ActionDeliveryMode,
ActionEffect,
@@ -180,6 +195,21 @@ _NativeCuaDriver.create_private_worker = classmethod(_create_private_worker_pyth
CuaDriver = _NativeCuaDriver
__all__ = [
+ "AppInfo",
+ "ClickPosition",
+ "ElementFrame",
+ "GetWindowStateInput",
+ "InputDeliveryMode",
+ "ListAppsInput",
+ "ListAppsOutput",
+ "ListWindowsInput",
+ "ListWindowsOutput",
+ "SnapshotImage",
+ "WindowBounds",
+ "WindowElement",
+ "WindowInfo",
+ "WindowStateOutput",
+
"ActionCompletion",
"ActionDelivery",
"ActionDeliveryMode",
diff --git a/libs/cua-driver/python/src/cua_driver/_native.py b/libs/cua-driver/python/src/cua_driver/_native.py
index ad55f69513..53eea7b8b5 100644
--- a/libs/cua-driver/python/src/cua_driver/_native.py
+++ b/libs/cua-driver/python/src/cua_driver/_native.py
@@ -522,7 +522,7 @@ def _uniffi_check_api_checksums(lib):
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_call_tool() != 24493:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
- if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_click() != 56077:
+ if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_click() != 22807:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_clipboard_read() != 58238:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
@@ -548,18 +548,24 @@ def _uniffi_check_api_checksums(lib):
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_get_session_state() != 49966:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
+ if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_get_window_state() != 62142:
+ raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_hotkey() != 53333:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_invoke_menu() != 13118:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_is_available() != 42961:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
+ if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_apps() != 35505:
+ raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_host_sessions_json() != 13193:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_sessions() != 40131:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_tools_json() != 33039:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
+ if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_windows() != 52601:
+ raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_metadata() != 55026:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriver_move_cursor() != 18320:
@@ -590,7 +596,7 @@ def _uniffi_check_api_checksums(lib):
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_call_tool() != 52859:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
- if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_click() != 22303:
+ if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_click() != 50816:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_clipboard_read() != 23560:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
@@ -616,12 +622,18 @@ def _uniffi_check_api_checksums(lib):
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_session_state() != 55370:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
+ if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_window_state() != 21266:
+ raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_hotkey() != 12820:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_invoke_menu() != 2742:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
+ if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_apps() != 51520:
+ raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_sessions() != 32840:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
+ if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_windows() != 20571:
+ raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_move_cursor() != 35768:
raise InternalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project")
if lib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_press_key() != 36768:
@@ -1259,6 +1271,11 @@ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_get_session_state.argtypes
cua_driver._native_contract._UniffiRustBuffer,
)
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_get_session_state.restype = ctypes.c_uint64
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_get_window_state.argtypes = (
+ ctypes.c_uint64,
+ cua_driver._native_contract._UniffiRustBuffer,
+)
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_get_window_state.restype = ctypes.c_uint64
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_hotkey.argtypes = (
ctypes.c_uint64,
cua_driver._native_contract._UniffiRustBuffer,
@@ -1274,6 +1291,11 @@ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_is_available.argtypes = (
ctypes.POINTER(_UniffiRustCallStatus),
)
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_is_available.restype = ctypes.c_int8
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_apps.argtypes = (
+ ctypes.c_uint64,
+ cua_driver._native_contract._UniffiRustBuffer,
+)
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_apps.restype = ctypes.c_uint64
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_host_sessions_json.argtypes = (
ctypes.c_uint64,
)
@@ -1287,6 +1309,11 @@ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_tools_json.argtypes =
ctypes.c_uint64,
)
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_tools_json.restype = ctypes.c_uint64
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_windows.argtypes = (
+ ctypes.c_uint64,
+ cua_driver._native_contract._UniffiRustBuffer,
+)
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_windows.restype = ctypes.c_uint64
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_metadata.argtypes = (
ctypes.c_uint64,
)
@@ -1426,6 +1453,11 @@ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_get_session_state.ar
cua_driver._native_contract._UniffiRustBuffer,
)
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_get_session_state.restype = ctypes.c_uint64
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_get_window_state.argtypes = (
+ ctypes.c_uint64,
+ cua_driver._native_contract._UniffiRustBuffer,
+)
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_get_window_state.restype = ctypes.c_uint64
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_hotkey.argtypes = (
ctypes.c_uint64,
cua_driver._native_contract._UniffiRustBuffer,
@@ -1436,11 +1468,21 @@ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_invoke_menu.argtypes
cua_driver._native_contract._UniffiRustBuffer,
)
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_invoke_menu.restype = ctypes.c_uint64
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_list_apps.argtypes = (
+ ctypes.c_uint64,
+ cua_driver._native_contract._UniffiRustBuffer,
+)
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_list_apps.restype = ctypes.c_uint64
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_list_sessions.argtypes = (
ctypes.c_uint64,
cua_driver._native_contract._UniffiRustBuffer,
)
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_list_sessions.restype = ctypes.c_uint64
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_list_windows.argtypes = (
+ ctypes.c_uint64,
+ cua_driver._native_contract._UniffiRustBuffer,
+)
+_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_list_windows.restype = ctypes.c_uint64
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_move_cursor.argtypes = (
ctypes.c_uint64,
cua_driver._native_contract._UniffiRustBuffer,
@@ -1673,6 +1715,9 @@ _UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_get_session.restype =
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_get_session_state.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_get_session_state.restype = ctypes.c_uint16
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_get_window_state.argtypes = (
+)
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_get_window_state.restype = ctypes.c_uint16
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_hotkey.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_hotkey.restype = ctypes.c_uint16
@@ -1682,6 +1727,9 @@ _UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_invoke_menu.restype =
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_is_available.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_is_available.restype = ctypes.c_uint16
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_apps.argtypes = (
+)
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_apps.restype = ctypes.c_uint16
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_host_sessions_json.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_host_sessions_json.restype = ctypes.c_uint16
@@ -1691,6 +1739,9 @@ _UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_sessions.restype
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_tools_json.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_tools_json.restype = ctypes.c_uint16
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_windows.argtypes = (
+)
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_list_windows.restype = ctypes.c_uint16
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_metadata.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriver_metadata.restype = ctypes.c_uint16
@@ -1775,15 +1826,24 @@ _UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_session.re
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_session_state.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_session_state.restype = ctypes.c_uint16
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_window_state.argtypes = (
+)
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_window_state.restype = ctypes.c_uint16
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_hotkey.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_hotkey.restype = ctypes.c_uint16
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_invoke_menu.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_invoke_menu.restype = ctypes.c_uint16
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_apps.argtypes = (
+)
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_apps.restype = ctypes.c_uint16
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_sessions.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_sessions.restype = ctypes.c_uint16
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_windows.argtypes = (
+)
+_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_windows.restype = ctypes.c_uint16
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_move_cursor.argtypes = (
)
_UniffiLib.uniffi_cua_driver_sdk_checksum_method_cuadriversession_move_cursor.restype = ctypes.c_uint16
@@ -5135,6 +5195,18 @@ class _UniffiFfiConverterTypeSdkClientKind(_UniffiConverterRustBuffer):
+
+
+
+
+
+
+
+
+
+
+
+
@@ -5157,7 +5229,7 @@ class CuaDriverProtocol(typing.Protocol):
remain downstream of the same public SDK runtime.
"""
raise NotImplementedError
- async def click(self, input: cua_driver._native_contract.ClickInput) -> ToolResult:
+ async def click(self, input: cua_driver._native_contract.ClickInput) -> cua_driver._native_contract.ActionResult:
raise NotImplementedError
async def clipboard_read(self, input: cua_driver._native_contract.ClipboardReadInput) -> ToolResult:
raise NotImplementedError
@@ -5183,12 +5255,16 @@ class CuaDriverProtocol(typing.Protocol):
raise NotImplementedError
async def get_session_state(self, input: cua_driver._native_contract.GetSessionStateInput) -> cua_driver._native_contract.SessionStateOutput:
raise NotImplementedError
+ async def get_window_state(self, input: cua_driver._native_contract.GetWindowStateInput) -> cua_driver._native_contract.WindowStateOutput:
+ raise NotImplementedError
async def hotkey(self, input: cua_driver._native_contract.HotkeyInput) -> ToolResult:
raise NotImplementedError
async def invoke_menu(self, input: cua_driver._native_contract.InvokeMenuInput) -> ToolResult:
raise NotImplementedError
def is_available(self, ) -> bool:
raise NotImplementedError
+ async def list_apps(self, input: cua_driver._native_contract.ListAppsInput) -> cua_driver._native_contract.ListAppsOutput:
+ raise NotImplementedError
async def list_host_sessions_json(self, ) -> str:
"""
Content-free lifecycle summaries for the trusted host's runtime or
@@ -5203,6 +5279,8 @@ class CuaDriverProtocol(typing.Protocol):
Canonical tool inventory for MCP and other protocol adapters.
"""
raise NotImplementedError
+ async def list_windows(self, input: cua_driver._native_contract.ListWindowsInput) -> cua_driver._native_contract.ListWindowsOutput:
+ raise NotImplementedError
async def metadata(self, ) -> DriverMetadata:
raise NotImplementedError
async def move_cursor(self, input: cua_driver._native_contract.MoveCursorInput) -> ToolResult:
@@ -5604,14 +5682,14 @@ class CuaDriver(CuaDriverProtocol):
_uniffi_lift_return,
_uniffi_error_converter,
)
- async def click(self, input: cua_driver._native_contract.ClickInput) -> ToolResult:
+ async def click(self, input: cua_driver._native_contract.ClickInput) -> cua_driver._native_contract.ActionResult:
cua_driver._native_contract._UniffiFfiConverterTypeClickInput.check_lower(input)
_uniffi_lowered_args = (
self._uniffi_clone_handle(),
cua_driver._native_contract._UniffiFfiConverterTypeClickInput.lower(input),
)
- _uniffi_lift_return = _UniffiFfiConverterTypeToolResult.lift
+ _uniffi_lift_return = cua_driver._native_contract._UniffiFfiConverterTypeActionResult.lift
_uniffi_error_converter = _UniffiFfiConverterTypeDriverError
return await _uniffi_rust_call_async(
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_click(*_uniffi_lowered_args),
@@ -5820,6 +5898,23 @@ class CuaDriver(CuaDriverProtocol):
_uniffi_lift_return,
_uniffi_error_converter,
)
+ async def get_window_state(self, input: cua_driver._native_contract.GetWindowStateInput) -> cua_driver._native_contract.WindowStateOutput:
+
+ cua_driver._native_contract._UniffiFfiConverterTypeGetWindowStateInput.check_lower(input)
+ _uniffi_lowered_args = (
+ self._uniffi_clone_handle(),
+ cua_driver._native_contract._UniffiFfiConverterTypeGetWindowStateInput.lower(input),
+ )
+ _uniffi_lift_return = cua_driver._native_contract._UniffiFfiConverterTypeWindowStateOutput.lift
+ _uniffi_error_converter = _UniffiFfiConverterTypeDriverError
+ return await _uniffi_rust_call_async(
+ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_get_window_state(*_uniffi_lowered_args),
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ _uniffi_lift_return,
+ _uniffi_error_converter,
+ )
async def hotkey(self, input: cua_driver._native_contract.HotkeyInput) -> ToolResult:
cua_driver._native_contract._UniffiFfiConverterTypeHotkeyInput.check_lower(input)
@@ -5866,6 +5961,23 @@ class CuaDriver(CuaDriverProtocol):
*_uniffi_lowered_args,
)
return _uniffi_lift_return(_uniffi_ffi_result)
+ async def list_apps(self, input: cua_driver._native_contract.ListAppsInput) -> cua_driver._native_contract.ListAppsOutput:
+
+ cua_driver._native_contract._UniffiFfiConverterTypeListAppsInput.check_lower(input)
+ _uniffi_lowered_args = (
+ self._uniffi_clone_handle(),
+ cua_driver._native_contract._UniffiFfiConverterTypeListAppsInput.lower(input),
+ )
+ _uniffi_lift_return = cua_driver._native_contract._UniffiFfiConverterTypeListAppsOutput.lift
+ _uniffi_error_converter = _UniffiFfiConverterTypeDriverError
+ return await _uniffi_rust_call_async(
+ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_apps(*_uniffi_lowered_args),
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ _uniffi_lift_return,
+ _uniffi_error_converter,
+ )
async def list_host_sessions_json(self, ) -> str:
"""
Content-free lifecycle summaries for the trusted host's runtime or
@@ -5919,6 +6031,23 @@ class CuaDriver(CuaDriverProtocol):
_uniffi_lift_return,
_uniffi_error_converter,
)
+ async def list_windows(self, input: cua_driver._native_contract.ListWindowsInput) -> cua_driver._native_contract.ListWindowsOutput:
+
+ cua_driver._native_contract._UniffiFfiConverterTypeListWindowsInput.check_lower(input)
+ _uniffi_lowered_args = (
+ self._uniffi_clone_handle(),
+ cua_driver._native_contract._UniffiFfiConverterTypeListWindowsInput.lower(input),
+ )
+ _uniffi_lift_return = cua_driver._native_contract._UniffiFfiConverterTypeListWindowsOutput.lift
+ _uniffi_error_converter = _UniffiFfiConverterTypeDriverError
+ return await _uniffi_rust_call_async(
+ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriver_list_windows(*_uniffi_lowered_args),
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ _uniffi_lift_return,
+ _uniffi_error_converter,
+ )
async def metadata(self, ) -> DriverMetadata:
_uniffi_lowered_args = (
self._uniffi_clone_handle(),
@@ -6184,7 +6313,7 @@ class CuaDriverSessionProtocol(typing.Protocol):
async def call_tool(self, name: str,arguments_json: str) -> ToolResult:
raise NotImplementedError
- async def click(self, input: cua_driver._native_contract.ClickInput) -> ToolResult:
+ async def click(self, input: cua_driver._native_contract.ClickInput) -> cua_driver._native_contract.ActionResult:
raise NotImplementedError
async def clipboard_read(self, input: cua_driver._native_contract.ClipboardReadInput) -> ToolResult:
raise NotImplementedError
@@ -6216,12 +6345,18 @@ class CuaDriverSessionProtocol(typing.Protocol):
raise NotImplementedError
async def get_session_state(self, input: cua_driver._native_contract.GetSessionStateInput) -> cua_driver._native_contract.SessionStateOutput:
raise NotImplementedError
+ async def get_window_state(self, input: cua_driver._native_contract.GetWindowStateInput) -> cua_driver._native_contract.WindowStateOutput:
+ raise NotImplementedError
async def hotkey(self, input: cua_driver._native_contract.HotkeyInput) -> ToolResult:
raise NotImplementedError
async def invoke_menu(self, input: cua_driver._native_contract.InvokeMenuInput) -> ToolResult:
raise NotImplementedError
+ async def list_apps(self, input: cua_driver._native_contract.ListAppsInput) -> cua_driver._native_contract.ListAppsOutput:
+ raise NotImplementedError
async def list_sessions(self, input: cua_driver._native_contract.ListSessionsInput) -> cua_driver._native_contract.ListSessionsOutput:
raise NotImplementedError
+ async def list_windows(self, input: cua_driver._native_contract.ListWindowsInput) -> cua_driver._native_contract.ListWindowsOutput:
+ raise NotImplementedError
async def move_cursor(self, input: cua_driver._native_contract.MoveCursorInput) -> ToolResult:
raise NotImplementedError
async def press_key(self, input: cua_driver._native_contract.PressKeyInput) -> ToolResult:
@@ -6287,14 +6422,14 @@ class CuaDriverSession(CuaDriverSessionProtocol):
_uniffi_lift_return,
_uniffi_error_converter,
)
- async def click(self, input: cua_driver._native_contract.ClickInput) -> ToolResult:
+ async def click(self, input: cua_driver._native_contract.ClickInput) -> cua_driver._native_contract.ActionResult:
cua_driver._native_contract._UniffiFfiConverterTypeClickInput.check_lower(input)
_uniffi_lowered_args = (
self._uniffi_clone_handle(),
cua_driver._native_contract._UniffiFfiConverterTypeClickInput.lower(input),
)
- _uniffi_lift_return = _UniffiFfiConverterTypeToolResult.lift
+ _uniffi_lift_return = cua_driver._native_contract._UniffiFfiConverterTypeActionResult.lift
_uniffi_error_converter = _UniffiFfiConverterTypeDriverError
return await _uniffi_rust_call_async(
_UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_click(*_uniffi_lowered_args),
@@ -6509,6 +6644,23 @@ class CuaDriverSession(CuaDriverSessionProtocol):
_uniffi_lift_return,
_uniffi_error_converter,
)
+ async def get_window_state(self, input: cua_driver._native_contract.GetWindowStateInput) -> cua_driver._native_contract.WindowStateOutput:
+
+ cua_driver._native_contract._UniffiFfiConverterTypeGetWindowStateInput.check_lower(input)
+ _uniffi_lowered_args = (
+ self._uniffi_clone_handle(),
+ cua_driver._native_contract._UniffiFfiConverterTypeGetWindowStateInput.lower(input),
+ )
+ _uniffi_lift_return = cua_driver._native_contract._UniffiFfiConverterTypeWindowStateOutput.lift
+ _uniffi_error_converter = _UniffiFfiConverterTypeDriverError
+ return await _uniffi_rust_call_async(
+ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_get_window_state(*_uniffi_lowered_args),
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ _uniffi_lift_return,
+ _uniffi_error_converter,
+ )
async def hotkey(self, input: cua_driver._native_contract.HotkeyInput) -> ToolResult:
cua_driver._native_contract._UniffiFfiConverterTypeHotkeyInput.check_lower(input)
@@ -6543,6 +6695,23 @@ class CuaDriverSession(CuaDriverSessionProtocol):
_uniffi_lift_return,
_uniffi_error_converter,
)
+ async def list_apps(self, input: cua_driver._native_contract.ListAppsInput) -> cua_driver._native_contract.ListAppsOutput:
+
+ cua_driver._native_contract._UniffiFfiConverterTypeListAppsInput.check_lower(input)
+ _uniffi_lowered_args = (
+ self._uniffi_clone_handle(),
+ cua_driver._native_contract._UniffiFfiConverterTypeListAppsInput.lower(input),
+ )
+ _uniffi_lift_return = cua_driver._native_contract._UniffiFfiConverterTypeListAppsOutput.lift
+ _uniffi_error_converter = _UniffiFfiConverterTypeDriverError
+ return await _uniffi_rust_call_async(
+ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_list_apps(*_uniffi_lowered_args),
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ _uniffi_lift_return,
+ _uniffi_error_converter,
+ )
async def list_sessions(self, input: cua_driver._native_contract.ListSessionsInput) -> cua_driver._native_contract.ListSessionsOutput:
cua_driver._native_contract._UniffiFfiConverterTypeListSessionsInput.check_lower(input)
@@ -6560,6 +6729,23 @@ class CuaDriverSession(CuaDriverSessionProtocol):
_uniffi_lift_return,
_uniffi_error_converter,
)
+ async def list_windows(self, input: cua_driver._native_contract.ListWindowsInput) -> cua_driver._native_contract.ListWindowsOutput:
+
+ cua_driver._native_contract._UniffiFfiConverterTypeListWindowsInput.check_lower(input)
+ _uniffi_lowered_args = (
+ self._uniffi_clone_handle(),
+ cua_driver._native_contract._UniffiFfiConverterTypeListWindowsInput.lower(input),
+ )
+ _uniffi_lift_return = cua_driver._native_contract._UniffiFfiConverterTypeListWindowsOutput.lift
+ _uniffi_error_converter = _UniffiFfiConverterTypeDriverError
+ return await _uniffi_rust_call_async(
+ _UniffiLib.uniffi_cua_driver_sdk_fn_method_cuadriversession_list_windows(*_uniffi_lowered_args),
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ _UniffiLib.ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ _uniffi_lift_return,
+ _uniffi_error_converter,
+ )
async def move_cursor(self, input: cua_driver._native_contract.MoveCursorInput) -> ToolResult:
cua_driver._native_contract._UniffiFfiConverterTypeMoveCursorInput.check_lower(input)
diff --git a/libs/cua-driver/python/src/cua_driver/_native_contract.py b/libs/cua-driver/python/src/cua_driver/_native_contract.py
index 439a467d63..2246229c17 100644
--- a/libs/cua-driver/python/src/cua_driver/_native_contract.py
+++ b/libs/cua-driver/python/src/cua_driver/_native_contract.py
@@ -1398,6 +1398,156 @@ class _UniffiFfiConverterTypeActionResult(_UniffiConverterRustBuffer):
_UniffiFfiConverterOptionalSequenceTypeActionEvidence.write(value.evidence, buf)
_UniffiFfiConverterOptionalTypeActionEscalation.write(value.escalation, buf)
+class _UniffiFfiConverterString:
+ @staticmethod
+ def check_lower(value):
+ if not isinstance(value, str):
+ raise TypeError("argument must be str, not {}".format(type(value).__name__))
+ return value
+
+ @staticmethod
+ def read(buf):
+ size = buf.read_i32()
+ if size < 0:
+ raise InternalError("Unexpected negative string length")
+ utf8_bytes = buf.read(size)
+ return utf8_bytes.decode("utf-8")
+
+ @staticmethod
+ def write(value, buf):
+ utf8_bytes = value.encode("utf-8")
+ buf.write_i32(len(utf8_bytes))
+ buf.write(utf8_bytes)
+
+ @staticmethod
+ def lift(buf):
+ with buf.consume_with_stream() as stream:
+ return stream.read(stream.remaining()).decode("utf-8")
+
+ @staticmethod
+ def lower(value):
+ with _UniffiRustBuffer.alloc_with_builder() as builder:
+ builder.write(value.encode("utf-8"))
+ return builder.finalize()
+
+class _UniffiFfiConverterBoolean:
+ @classmethod
+ def check_lower(cls, value):
+ return not not value
+
+ @classmethod
+ def lower(cls, value):
+ return 1 if value else 0
+
+ @staticmethod
+ def lift(value):
+ return value != 0
+
+ @classmethod
+ def read(cls, buf):
+ return cls.lift(buf.read_u8())
+
+ @classmethod
+ def write(cls, value, buf):
+ buf.write_u8(value)
+
+class _UniffiFfiConverterOptionalString(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterString.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterString.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterString.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
+@dataclass
+class AppInfo:
+ def __init__(self, *, pid:int, name:str, running:bool, active:bool, bundle_id:typing.Optional[str], launch_path:typing.Optional[str], kind:typing.Optional[str], last_used:typing.Optional[str]):
+ self.pid = pid
+ self.name = name
+ self.running = running
+ self.active = active
+ self.bundle_id = bundle_id
+ self.launch_path = launch_path
+ self.kind = kind
+ self.last_used = last_used
+
+
+
+
+ def __str__(self):
+ return "AppInfo(pid={}, name={}, running={}, active={}, bundle_id={}, launch_path={}, kind={}, last_used={})".format(self.pid, self.name, self.running, self.active, self.bundle_id, self.launch_path, self.kind, self.last_used)
+ def __eq__(self, other):
+ if self.pid != other.pid:
+ return False
+ if self.name != other.name:
+ return False
+ if self.running != other.running:
+ return False
+ if self.active != other.active:
+ return False
+ if self.bundle_id != other.bundle_id:
+ return False
+ if self.launch_path != other.launch_path:
+ return False
+ if self.kind != other.kind:
+ return False
+ if self.last_used != other.last_used:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeAppInfo(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return AppInfo(
+ pid=_UniffiFfiConverterUInt32.read(buf),
+ name=_UniffiFfiConverterString.read(buf),
+ running=_UniffiFfiConverterBoolean.read(buf),
+ active=_UniffiFfiConverterBoolean.read(buf),
+ bundle_id=_UniffiFfiConverterOptionalString.read(buf),
+ launch_path=_UniffiFfiConverterOptionalString.read(buf),
+ kind=_UniffiFfiConverterOptionalString.read(buf),
+ last_used=_UniffiFfiConverterOptionalString.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterUInt32.check_lower(value.pid)
+ _UniffiFfiConverterString.check_lower(value.name)
+ _UniffiFfiConverterBoolean.check_lower(value.running)
+ _UniffiFfiConverterBoolean.check_lower(value.active)
+ _UniffiFfiConverterOptionalString.check_lower(value.bundle_id)
+ _UniffiFfiConverterOptionalString.check_lower(value.launch_path)
+ _UniffiFfiConverterOptionalString.check_lower(value.kind)
+ _UniffiFfiConverterOptionalString.check_lower(value.last_used)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterUInt32.write(value.pid, buf)
+ _UniffiFfiConverterString.write(value.name, buf)
+ _UniffiFfiConverterBoolean.write(value.running, buf)
+ _UniffiFfiConverterBoolean.write(value.active, buf)
+ _UniffiFfiConverterOptionalString.write(value.bundle_id, buf)
+ _UniffiFfiConverterOptionalString.write(value.launch_path, buf)
+ _UniffiFfiConverterOptionalString.write(value.kind, buf)
+ _UniffiFfiConverterOptionalString.write(value.last_used, buf)
+
class _UniffiFfiConverterFloat64(_UniffiConverterPrimitiveFloat):
@staticmethod
def read(buf):
@@ -1499,38 +1649,6 @@ class _UniffiFfiConverterUInt64(_UniffiConverterPrimitiveInt):
def write(value, buf):
buf.write_u64(value)
-class _UniffiFfiConverterString:
- @staticmethod
- def check_lower(value):
- if not isinstance(value, str):
- raise TypeError("argument must be str, not {}".format(type(value).__name__))
- return value
-
- @staticmethod
- def read(buf):
- size = buf.read_i32()
- if size < 0:
- raise InternalError("Unexpected negative string length")
- utf8_bytes = buf.read(size)
- return utf8_bytes.decode("utf-8")
-
- @staticmethod
- def write(value, buf):
- utf8_bytes = value.encode("utf-8")
- buf.write_i32(len(utf8_bytes))
- buf.write(utf8_bytes)
-
- @staticmethod
- def lift(buf):
- with buf.consume_with_stream() as stream:
- return stream.read(stream.remaining()).decode("utf-8")
-
- @staticmethod
- def lower(value):
- with _UniffiRustBuffer.alloc_with_builder() as builder:
- builder.write(value.encode("utf-8"))
- return builder.finalize()
-
@@ -1662,112 +1780,169 @@ class _UniffiFfiConverterTypeActionTarget(_UniffiConverterRustBuffer):
-class _UniffiFfiConverterOptionalTypeActionTarget(_UniffiConverterRustBuffer):
- @classmethod
- def check_lower(cls, value):
- if value is not None:
- _UniffiFfiConverterTypeActionTarget.check_lower(value)
- @classmethod
- def write(cls, value, buf):
- if value is None:
- buf.write_u8(0)
- return
- buf.write_u8(1)
- _UniffiFfiConverterTypeActionTarget.write(value, buf)
- @classmethod
- def read(cls, buf):
- flag = buf.read_u8()
- if flag == 0:
- return None
- elif flag == 1:
- return _UniffiFfiConverterTypeActionTarget.read(buf)
- else:
- raise InternalError("Unexpected flag byte for optional type")
+class ClickPosition:
+ def __init__(self):
+ raise RuntimeError("ClickPosition cannot be instantiated directly")
+
+ # Each enum variant is a nested class of the enum itself.
+ @dataclass
+ class COORDINATES:
+ def __init__(self, x:float, y:float):
+ self.x = x
+ self.y = y
-class DesktopScope(enum.Enum):
- DESKTOP = 0
+ pass
-class _UniffiFfiConverterTypeDesktopScope(_UniffiConverterRustBuffer):
+
+
+ def __str__(self):
+ return "ClickPosition.COORDINATES(x={}, y={})".format(self.x, self.y)
+ def __eq__(self, other):
+ if not isinstance(other, ClickPosition):
+ return NotImplemented
+ if not other.is_COORDINATES():
+ return False
+ if self.x != other.x:
+ return False
+ if self.y != other.y:
+ return False
+ return True
+
+ @dataclass
+ class ELEMENT:
+
+ def __init__(self, element_token:str):
+ self.element_token = element_token
+
+
+ pass
+
+
+
+
+
+ def __str__(self):
+ return "ClickPosition.ELEMENT(element_token={})".format(self.element_token)
+ def __eq__(self, other):
+ if not isinstance(other, ClickPosition):
+ return NotImplemented
+ if not other.is_ELEMENT():
+ return False
+ if self.element_token != other.element_token:
+ return False
+ return True
+
+
+
+ # For each variant, we have `is_NAME` and `is_name` methods for easily checking
+ # whether an instance is that variant.
+ def is_COORDINATES(self) -> bool:
+ return isinstance(self, ClickPosition.COORDINATES)
+ def is_coordinates(self) -> bool:
+ return isinstance(self, ClickPosition.COORDINATES)
+ def is_ELEMENT(self) -> bool:
+ return isinstance(self, ClickPosition.ELEMENT)
+ def is_element(self) -> bool:
+ return isinstance(self, ClickPosition.ELEMENT)
+
+
+# Now, a little trick - we make each nested variant class be a subclass of the main
+# enum class, so that method calls and instance checks etc will work intuitively.
+# We might be able to do this a little more neatly with a metaclass, but this'll do.
+ClickPosition.COORDINATES = type("ClickPosition.COORDINATES", (ClickPosition.COORDINATES, ClickPosition,), {}) # type: ignore
+ClickPosition.ELEMENT = type("ClickPosition.ELEMENT", (ClickPosition.ELEMENT, ClickPosition,), {}) # type: ignore
+
+
+
+
+class _UniffiFfiConverterTypeClickPosition(_UniffiConverterRustBuffer):
@staticmethod
def read(buf):
variant = buf.read_i32()
if variant == 1:
- return DesktopScope.DESKTOP
+ return ClickPosition.COORDINATES(
+ _UniffiFfiConverterFloat64.read(buf),
+ _UniffiFfiConverterFloat64.read(buf),
+ )
+ if variant == 2:
+ return ClickPosition.ELEMENT(
+ _UniffiFfiConverterString.read(buf),
+ )
raise InternalError("Raw enum value doesn't match any cases")
@staticmethod
def check_lower(value):
- if value == DesktopScope.DESKTOP:
+ if value.is_COORDINATES():
+ _UniffiFfiConverterFloat64.check_lower(value.x)
+ _UniffiFfiConverterFloat64.check_lower(value.y)
+ return
+ if value.is_ELEMENT():
+ _UniffiFfiConverterString.check_lower(value.element_token)
return
raise ValueError(value)
@staticmethod
def write(value, buf):
- if value == DesktopScope.DESKTOP:
+ if value.is_COORDINATES():
buf.write_i32(1)
+ _UniffiFfiConverterFloat64.write(value.x, buf)
+ _UniffiFfiConverterFloat64.write(value.y, buf)
+ if value.is_ELEMENT():
+ buf.write_i32(2)
+ _UniffiFfiConverterString.write(value.element_token, buf)
-class _UniffiFfiConverterOptionalTypeDesktopScope(_UniffiConverterRustBuffer):
- @classmethod
- def check_lower(cls, value):
- if value is not None:
- _UniffiFfiConverterTypeDesktopScope.check_lower(value)
- @classmethod
- def write(cls, value, buf):
- if value is None:
- buf.write_u8(0)
- return
- buf.write_u8(1)
- _UniffiFfiConverterTypeDesktopScope.write(value, buf)
- @classmethod
- def read(cls, buf):
- flag = buf.read_u8()
- if flag == 0:
- return None
- elif flag == 1:
- return _UniffiFfiConverterTypeDesktopScope.read(buf)
- else:
- raise InternalError("Unexpected flag byte for optional type")
-class _UniffiFfiConverterOptionalString(_UniffiConverterRustBuffer):
- @classmethod
- def check_lower(cls, value):
- if value is not None:
- _UniffiFfiConverterString.check_lower(value)
- @classmethod
- def write(cls, value, buf):
- if value is None:
- buf.write_u8(0)
+class InputDeliveryMode(enum.Enum):
+
+ BACKGROUND = 0
+
+ FOREGROUND = 1
+
+
+
+class _UniffiFfiConverterTypeInputDeliveryMode(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ variant = buf.read_i32()
+ if variant == 1:
+ return InputDeliveryMode.BACKGROUND
+ if variant == 2:
+ return InputDeliveryMode.FOREGROUND
+ raise InternalError("Raw enum value doesn't match any cases")
+
+ @staticmethod
+ def check_lower(value):
+ if value == InputDeliveryMode.BACKGROUND:
+ return
+ if value == InputDeliveryMode.FOREGROUND:
return
+ raise ValueError(value)
+
+ @staticmethod
+ def write(value, buf):
+ if value == InputDeliveryMode.BACKGROUND:
+ buf.write_i32(1)
+ if value == InputDeliveryMode.FOREGROUND:
+ buf.write_i32(2)
- buf.write_u8(1)
- _UniffiFfiConverterString.write(value, buf)
- @classmethod
- def read(cls, buf):
- flag = buf.read_u8()
- if flag == 0:
- return None
- elif flag == 1:
- return _UniffiFfiConverterString.read(buf)
- else:
- raise InternalError("Unexpected flag byte for optional type")
@@ -1844,11 +2019,10 @@ class _UniffiFfiConverterOptionalTypeClickButton(_UniffiConverterRustBuffer):
@dataclass
class ClickInput:
- def __init__(self, *, x:float, y:float, target:typing.Optional[ActionTarget], scope:typing.Optional[DesktopScope], session:typing.Optional[str], button:typing.Optional[ClickButton], count:typing.Optional[int]):
- self.x = x
- self.y = y
+ def __init__(self, *, target:ActionTarget, position:ClickPosition, delivery_mode:InputDeliveryMode, session:typing.Optional[str], button:typing.Optional[ClickButton], count:typing.Optional[int]):
self.target = target
- self.scope = scope
+ self.position = position
+ self.delivery_mode = delivery_mode
self.session = session
self.button = button
self.count = count
@@ -1857,15 +2031,13 @@ class ClickInput:
def __str__(self):
- return "ClickInput(x={}, y={}, target={}, scope={}, session={}, button={}, count={})".format(self.x, self.y, self.target, self.scope, self.session, self.button, self.count)
+ return "ClickInput(target={}, position={}, delivery_mode={}, session={}, button={}, count={})".format(self.target, self.position, self.delivery_mode, self.session, self.button, self.count)
def __eq__(self, other):
- if self.x != other.x:
- return False
- if self.y != other.y:
- return False
if self.target != other.target:
return False
- if self.scope != other.scope:
+ if self.position != other.position:
+ return False
+ if self.delivery_mode != other.delivery_mode:
return False
if self.session != other.session:
return False
@@ -1879,10 +2051,9 @@ class _UniffiFfiConverterTypeClickInput(_UniffiConverterRustBuffer):
@staticmethod
def read(buf):
return ClickInput(
- x=_UniffiFfiConverterFloat64.read(buf),
- y=_UniffiFfiConverterFloat64.read(buf),
- target=_UniffiFfiConverterOptionalTypeActionTarget.read(buf),
- scope=_UniffiFfiConverterOptionalTypeDesktopScope.read(buf),
+ target=_UniffiFfiConverterTypeActionTarget.read(buf),
+ position=_UniffiFfiConverterTypeClickPosition.read(buf),
+ delivery_mode=_UniffiFfiConverterTypeInputDeliveryMode.read(buf),
session=_UniffiFfiConverterOptionalString.read(buf),
button=_UniffiFfiConverterOptionalTypeClickButton.read(buf),
count=_UniffiFfiConverterOptionalUInt32.read(buf),
@@ -1890,45 +2061,22 @@ class _UniffiFfiConverterTypeClickInput(_UniffiConverterRustBuffer):
@staticmethod
def check_lower(value):
- _UniffiFfiConverterFloat64.check_lower(value.x)
- _UniffiFfiConverterFloat64.check_lower(value.y)
- _UniffiFfiConverterOptionalTypeActionTarget.check_lower(value.target)
- _UniffiFfiConverterOptionalTypeDesktopScope.check_lower(value.scope)
+ _UniffiFfiConverterTypeActionTarget.check_lower(value.target)
+ _UniffiFfiConverterTypeClickPosition.check_lower(value.position)
+ _UniffiFfiConverterTypeInputDeliveryMode.check_lower(value.delivery_mode)
_UniffiFfiConverterOptionalString.check_lower(value.session)
_UniffiFfiConverterOptionalTypeClickButton.check_lower(value.button)
_UniffiFfiConverterOptionalUInt32.check_lower(value.count)
@staticmethod
def write(value, buf):
- _UniffiFfiConverterFloat64.write(value.x, buf)
- _UniffiFfiConverterFloat64.write(value.y, buf)
- _UniffiFfiConverterOptionalTypeActionTarget.write(value.target, buf)
- _UniffiFfiConverterOptionalTypeDesktopScope.write(value.scope, buf)
+ _UniffiFfiConverterTypeActionTarget.write(value.target, buf)
+ _UniffiFfiConverterTypeClickPosition.write(value.position, buf)
+ _UniffiFfiConverterTypeInputDeliveryMode.write(value.delivery_mode, buf)
_UniffiFfiConverterOptionalString.write(value.session, buf)
_UniffiFfiConverterOptionalTypeClickButton.write(value.button, buf)
_UniffiFfiConverterOptionalUInt32.write(value.count, buf)
-class _UniffiFfiConverterBoolean:
- @classmethod
- def check_lower(cls, value):
- return not not value
-
- @classmethod
- def lower(cls, value):
- return 1 if value else 0
-
- @staticmethod
- def lift(value):
- return value != 0
-
- @classmethod
- def read(cls, buf):
- return cls.lift(buf.read_u8())
-
- @classmethod
- def write(cls, value, buf):
- buf.write_u8(value)
-
@dataclass
class ClipboardReadInput:
def __init__(self, *, include_text:bool, session:typing.Optional[str]):
@@ -2576,6 +2724,88 @@ class _UniffiFfiConverterTypeCursorVisualOutput(_UniffiConverterRustBuffer):
_UniffiFfiConverterUInt64.write(value.frame, buf)
_UniffiFfiConverterUInt64.write(value.preempted_count, buf)
+class _UniffiFfiConverterOptionalTypeActionTarget(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterTypeActionTarget.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterTypeActionTarget.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterTypeActionTarget.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
+
+
+
+
+
+class DesktopScope(enum.Enum):
+
+ DESKTOP = 0
+
+
+
+class _UniffiFfiConverterTypeDesktopScope(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ variant = buf.read_i32()
+ if variant == 1:
+ return DesktopScope.DESKTOP
+ raise InternalError("Raw enum value doesn't match any cases")
+
+ @staticmethod
+ def check_lower(value):
+ if value == DesktopScope.DESKTOP:
+ return
+ raise ValueError(value)
+
+ @staticmethod
+ def write(value, buf):
+ if value == DesktopScope.DESKTOP:
+ buf.write_i32(1)
+
+
+
+class _UniffiFfiConverterOptionalTypeDesktopScope(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterTypeDesktopScope.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterTypeDesktopScope.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterTypeDesktopScope.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
class _UniffiFfiConverterOptionalUInt64(_UniffiConverterRustBuffer):
@classmethod
def check_lower(cls, value):
@@ -2716,6 +2946,54 @@ class _UniffiFfiConverterTypeDragInput(_UniffiConverterRustBuffer):
_UniffiFfiConverterOptionalTypeClickButton.write(value.button, buf)
_UniffiFfiConverterOptionalSequenceString.write(value.modifier, buf)
+@dataclass
+class ElementFrame:
+ def __init__(self, *, x:float, y:float, w:float, h:float):
+ self.x = x
+ self.y = y
+ self.w = w
+ self.h = h
+
+
+
+
+ def __str__(self):
+ return "ElementFrame(x={}, y={}, w={}, h={})".format(self.x, self.y, self.w, self.h)
+ def __eq__(self, other):
+ if self.x != other.x:
+ return False
+ if self.y != other.y:
+ return False
+ if self.w != other.w:
+ return False
+ if self.h != other.h:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeElementFrame(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return ElementFrame(
+ x=_UniffiFfiConverterFloat64.read(buf),
+ y=_UniffiFfiConverterFloat64.read(buf),
+ w=_UniffiFfiConverterFloat64.read(buf),
+ h=_UniffiFfiConverterFloat64.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterFloat64.check_lower(value.x)
+ _UniffiFfiConverterFloat64.check_lower(value.y)
+ _UniffiFfiConverterFloat64.check_lower(value.w)
+ _UniffiFfiConverterFloat64.check_lower(value.h)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterFloat64.write(value.x, buf)
+ _UniffiFfiConverterFloat64.write(value.y, buf)
+ _UniffiFfiConverterFloat64.write(value.w, buf)
+ _UniffiFfiConverterFloat64.write(value.h, buf)
+
@dataclass
class ElementSelector:
def __init__(self, *, role:typing.Optional[str], label_contains:typing.Optional[str]):
@@ -3277,6 +3555,90 @@ class _UniffiFfiConverterTypeGetSessionStateInput(_UniffiConverterRustBuffer):
def write(value, buf):
_UniffiFfiConverterOptionalString.write(value.session, buf)
+@dataclass
+class GetWindowStateInput:
+ def __init__(self, *, pid:int, window_id:int, session:typing.Optional[str], query:typing.Optional[str], include_accessibility_tree:typing.Optional[bool], include_screenshot:typing.Optional[bool], screenshot_out_file:typing.Optional[str], max_elements:typing.Optional[int], max_depth:typing.Optional[int], max_dimension:typing.Optional[int]):
+ self.pid = pid
+ self.window_id = window_id
+ self.session = session
+ self.query = query
+ self.include_accessibility_tree = include_accessibility_tree
+ self.include_screenshot = include_screenshot
+ self.screenshot_out_file = screenshot_out_file
+ self.max_elements = max_elements
+ self.max_depth = max_depth
+ self.max_dimension = max_dimension
+
+
+
+
+ def __str__(self):
+ return "GetWindowStateInput(pid={}, window_id={}, session={}, query={}, include_accessibility_tree={}, include_screenshot={}, screenshot_out_file={}, max_elements={}, max_depth={}, max_dimension={})".format(self.pid, self.window_id, self.session, self.query, self.include_accessibility_tree, self.include_screenshot, self.screenshot_out_file, self.max_elements, self.max_depth, self.max_dimension)
+ def __eq__(self, other):
+ if self.pid != other.pid:
+ return False
+ if self.window_id != other.window_id:
+ return False
+ if self.session != other.session:
+ return False
+ if self.query != other.query:
+ return False
+ if self.include_accessibility_tree != other.include_accessibility_tree:
+ return False
+ if self.include_screenshot != other.include_screenshot:
+ return False
+ if self.screenshot_out_file != other.screenshot_out_file:
+ return False
+ if self.max_elements != other.max_elements:
+ return False
+ if self.max_depth != other.max_depth:
+ return False
+ if self.max_dimension != other.max_dimension:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeGetWindowStateInput(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return GetWindowStateInput(
+ pid=_UniffiFfiConverterUInt32.read(buf),
+ window_id=_UniffiFfiConverterUInt64.read(buf),
+ session=_UniffiFfiConverterOptionalString.read(buf),
+ query=_UniffiFfiConverterOptionalString.read(buf),
+ include_accessibility_tree=_UniffiFfiConverterOptionalBoolean.read(buf),
+ include_screenshot=_UniffiFfiConverterOptionalBoolean.read(buf),
+ screenshot_out_file=_UniffiFfiConverterOptionalString.read(buf),
+ max_elements=_UniffiFfiConverterOptionalUInt32.read(buf),
+ max_depth=_UniffiFfiConverterOptionalUInt32.read(buf),
+ max_dimension=_UniffiFfiConverterOptionalUInt32.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterUInt32.check_lower(value.pid)
+ _UniffiFfiConverterUInt64.check_lower(value.window_id)
+ _UniffiFfiConverterOptionalString.check_lower(value.session)
+ _UniffiFfiConverterOptionalString.check_lower(value.query)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.include_accessibility_tree)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.include_screenshot)
+ _UniffiFfiConverterOptionalString.check_lower(value.screenshot_out_file)
+ _UniffiFfiConverterOptionalUInt32.check_lower(value.max_elements)
+ _UniffiFfiConverterOptionalUInt32.check_lower(value.max_depth)
+ _UniffiFfiConverterOptionalUInt32.check_lower(value.max_dimension)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterUInt32.write(value.pid, buf)
+ _UniffiFfiConverterUInt64.write(value.window_id, buf)
+ _UniffiFfiConverterOptionalString.write(value.session, buf)
+ _UniffiFfiConverterOptionalString.write(value.query, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.include_accessibility_tree, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.include_screenshot, buf)
+ _UniffiFfiConverterOptionalString.write(value.screenshot_out_file, buf)
+ _UniffiFfiConverterOptionalUInt32.write(value.max_elements, buf)
+ _UniffiFfiConverterOptionalUInt32.write(value.max_depth, buf)
+ _UniffiFfiConverterOptionalUInt32.write(value.max_dimension, buf)
+
@dataclass
class HotkeyInput:
def __init__(self, *, keys:typing.List[str], target:typing.Optional[ActionTarget], scope:typing.Optional[DesktopScope], session:typing.Optional[str]):
@@ -3379,19 +3741,98 @@ class _UniffiFfiConverterTypeInvokeMenuInput(_UniffiConverterRustBuffer):
_UniffiFfiConverterOptionalString.write(value.session, buf)
@dataclass
-class ListSessionsInput:
- def __init__(self, *, limit:typing.Optional[int], cursor:typing.Optional[str]):
- self.limit = limit
- self.cursor = cursor
+class ListAppsInput:
+
def __str__(self):
- return "ListSessionsInput(limit={}, cursor={})".format(self.limit, self.cursor)
+ return "ListAppsInput()".format()
def __eq__(self, other):
- if self.limit != other.limit:
- return False
+ return True
+
+class _UniffiFfiConverterTypeListAppsInput(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return ListAppsInput(
+ )
+
+ @staticmethod
+ def check_lower(value):
+ pass
+
+ @staticmethod
+ def write(value, buf):
+ pass
+
+class _UniffiFfiConverterSequenceTypeAppInfo(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ for item in value:
+ _UniffiFfiConverterTypeAppInfo.check_lower(item)
+
+ @classmethod
+ def write(cls, value, buf):
+ items = len(value)
+ buf.write_i32(items)
+ for item in value:
+ _UniffiFfiConverterTypeAppInfo.write(item, buf)
+
+ @classmethod
+ def read(cls, buf):
+ count = buf.read_i32()
+ if count < 0:
+ raise InternalError("Unexpected negative sequence length")
+
+ return [
+ _UniffiFfiConverterTypeAppInfo.read(buf) for i in range(count)
+ ]
+
+@dataclass
+class ListAppsOutput:
+ def __init__(self, *, apps:typing.List[AppInfo]):
+ self.apps = apps
+
+
+
+
+ def __str__(self):
+ return "ListAppsOutput(apps={})".format(self.apps)
+ def __eq__(self, other):
+ if self.apps != other.apps:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeListAppsOutput(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return ListAppsOutput(
+ apps=_UniffiFfiConverterSequenceTypeAppInfo.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterSequenceTypeAppInfo.check_lower(value.apps)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterSequenceTypeAppInfo.write(value.apps, buf)
+
+@dataclass
+class ListSessionsInput:
+ def __init__(self, *, limit:typing.Optional[int], cursor:typing.Optional[str]):
+ self.limit = limit
+ self.cursor = cursor
+
+
+
+
+ def __str__(self):
+ return "ListSessionsInput(limit={}, cursor={})".format(self.limit, self.cursor)
+ def __eq__(self, other):
+ if self.limit != other.limit:
+ return False
if self.cursor != other.cursor:
return False
return True
@@ -3659,14 +4100,377 @@ class _UniffiFfiConverterSequenceTypeSessionOutput(_UniffiConverterRustBuffer):
@classmethod
def check_lower(cls, value):
for item in value:
- _UniffiFfiConverterTypeSessionOutput.check_lower(item)
+ _UniffiFfiConverterTypeSessionOutput.check_lower(item)
+
+ @classmethod
+ def write(cls, value, buf):
+ items = len(value)
+ buf.write_i32(items)
+ for item in value:
+ _UniffiFfiConverterTypeSessionOutput.write(item, buf)
+
+ @classmethod
+ def read(cls, buf):
+ count = buf.read_i32()
+ if count < 0:
+ raise InternalError("Unexpected negative sequence length")
+
+ return [
+ _UniffiFfiConverterTypeSessionOutput.read(buf) for i in range(count)
+ ]
+
+@dataclass
+class ListSessionsOutput:
+ def __init__(self, *, sessions:typing.List[SessionOutput], next_cursor:typing.Optional[str]):
+ self.sessions = sessions
+ self.next_cursor = next_cursor
+
+
+
+
+ def __str__(self):
+ return "ListSessionsOutput(sessions={}, next_cursor={})".format(self.sessions, self.next_cursor)
+ def __eq__(self, other):
+ if self.sessions != other.sessions:
+ return False
+ if self.next_cursor != other.next_cursor:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeListSessionsOutput(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return ListSessionsOutput(
+ sessions=_UniffiFfiConverterSequenceTypeSessionOutput.read(buf),
+ next_cursor=_UniffiFfiConverterOptionalString.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterSequenceTypeSessionOutput.check_lower(value.sessions)
+ _UniffiFfiConverterOptionalString.check_lower(value.next_cursor)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterSequenceTypeSessionOutput.write(value.sessions, buf)
+ _UniffiFfiConverterOptionalString.write(value.next_cursor, buf)
+
+@dataclass
+class ListWindowsInput:
+ def __init__(self, *, pid:typing.Optional[int], on_screen_only:typing.Optional[bool]):
+ self.pid = pid
+ self.on_screen_only = on_screen_only
+
+
+
+
+ def __str__(self):
+ return "ListWindowsInput(pid={}, on_screen_only={})".format(self.pid, self.on_screen_only)
+ def __eq__(self, other):
+ if self.pid != other.pid:
+ return False
+ if self.on_screen_only != other.on_screen_only:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeListWindowsInput(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return ListWindowsInput(
+ pid=_UniffiFfiConverterOptionalUInt32.read(buf),
+ on_screen_only=_UniffiFfiConverterOptionalBoolean.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterOptionalUInt32.check_lower(value.pid)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.on_screen_only)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterOptionalUInt32.write(value.pid, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.on_screen_only, buf)
+
+@dataclass
+class WindowBounds:
+ def __init__(self, *, x:float, y:float, width:float, height:float):
+ self.x = x
+ self.y = y
+ self.width = width
+ self.height = height
+
+
+
+
+ def __str__(self):
+ return "WindowBounds(x={}, y={}, width={}, height={})".format(self.x, self.y, self.width, self.height)
+ def __eq__(self, other):
+ if self.x != other.x:
+ return False
+ if self.y != other.y:
+ return False
+ if self.width != other.width:
+ return False
+ if self.height != other.height:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeWindowBounds(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return WindowBounds(
+ x=_UniffiFfiConverterFloat64.read(buf),
+ y=_UniffiFfiConverterFloat64.read(buf),
+ width=_UniffiFfiConverterFloat64.read(buf),
+ height=_UniffiFfiConverterFloat64.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterFloat64.check_lower(value.x)
+ _UniffiFfiConverterFloat64.check_lower(value.y)
+ _UniffiFfiConverterFloat64.check_lower(value.width)
+ _UniffiFfiConverterFloat64.check_lower(value.height)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterFloat64.write(value.x, buf)
+ _UniffiFfiConverterFloat64.write(value.y, buf)
+ _UniffiFfiConverterFloat64.write(value.width, buf)
+ _UniffiFfiConverterFloat64.write(value.height, buf)
+
+class _UniffiFfiConverterInt64(_UniffiConverterPrimitiveInt):
+ CLASS_NAME = "i64"
+ VALUE_MIN = -2**63
+ VALUE_MAX = 2**63
+
+ @staticmethod
+ def read(buf):
+ return buf.read_i64()
+
+ @staticmethod
+ def write(value, buf):
+ buf.write_i64(value)
+
+class _UniffiFfiConverterOptionalInt64(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterInt64.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterInt64.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterInt64.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
+class _UniffiFfiConverterInt32(_UniffiConverterPrimitiveInt):
+ CLASS_NAME = "i32"
+ VALUE_MIN = -2**31
+ VALUE_MAX = 2**31
+
+ @staticmethod
+ def read(buf):
+ return buf.read_i32()
+
+ @staticmethod
+ def write(value, buf):
+ buf.write_i32(value)
+
+class _UniffiFfiConverterOptionalInt32(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterInt32.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterInt32.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterInt32.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
+class _UniffiFfiConverterSequenceUInt64(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ for item in value:
+ _UniffiFfiConverterUInt64.check_lower(item)
+
+ @classmethod
+ def write(cls, value, buf):
+ items = len(value)
+ buf.write_i32(items)
+ for item in value:
+ _UniffiFfiConverterUInt64.write(item, buf)
+
+ @classmethod
+ def read(cls, buf):
+ count = buf.read_i32()
+ if count < 0:
+ raise InternalError("Unexpected negative sequence length")
+
+ return [
+ _UniffiFfiConverterUInt64.read(buf) for i in range(count)
+ ]
+
+class _UniffiFfiConverterOptionalSequenceUInt64(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterSequenceUInt64.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterSequenceUInt64.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterSequenceUInt64.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
+@dataclass
+class WindowInfo:
+ def __init__(self, *, window_id:int, pid:typing.Optional[int], app_name:str, title:str, bounds:WindowBounds, is_on_screen:bool, z_index:typing.Optional[int], layer:typing.Optional[int], minimized:typing.Optional[bool], current_space_id:typing.Optional[int], on_current_space:typing.Optional[bool], space_ids:typing.Optional[typing.List[int]]):
+ self.window_id = window_id
+ self.pid = pid
+ self.app_name = app_name
+ self.title = title
+ self.bounds = bounds
+ self.is_on_screen = is_on_screen
+ self.z_index = z_index
+ self.layer = layer
+ self.minimized = minimized
+ self.current_space_id = current_space_id
+ self.on_current_space = on_current_space
+ self.space_ids = space_ids
+
+
+
+
+ def __str__(self):
+ return "WindowInfo(window_id={}, pid={}, app_name={}, title={}, bounds={}, is_on_screen={}, z_index={}, layer={}, minimized={}, current_space_id={}, on_current_space={}, space_ids={})".format(self.window_id, self.pid, self.app_name, self.title, self.bounds, self.is_on_screen, self.z_index, self.layer, self.minimized, self.current_space_id, self.on_current_space, self.space_ids)
+ def __eq__(self, other):
+ if self.window_id != other.window_id:
+ return False
+ if self.pid != other.pid:
+ return False
+ if self.app_name != other.app_name:
+ return False
+ if self.title != other.title:
+ return False
+ if self.bounds != other.bounds:
+ return False
+ if self.is_on_screen != other.is_on_screen:
+ return False
+ if self.z_index != other.z_index:
+ return False
+ if self.layer != other.layer:
+ return False
+ if self.minimized != other.minimized:
+ return False
+ if self.current_space_id != other.current_space_id:
+ return False
+ if self.on_current_space != other.on_current_space:
+ return False
+ if self.space_ids != other.space_ids:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeWindowInfo(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return WindowInfo(
+ window_id=_UniffiFfiConverterUInt64.read(buf),
+ pid=_UniffiFfiConverterOptionalUInt32.read(buf),
+ app_name=_UniffiFfiConverterString.read(buf),
+ title=_UniffiFfiConverterString.read(buf),
+ bounds=_UniffiFfiConverterTypeWindowBounds.read(buf),
+ is_on_screen=_UniffiFfiConverterBoolean.read(buf),
+ z_index=_UniffiFfiConverterOptionalInt64.read(buf),
+ layer=_UniffiFfiConverterOptionalInt32.read(buf),
+ minimized=_UniffiFfiConverterOptionalBoolean.read(buf),
+ current_space_id=_UniffiFfiConverterOptionalUInt64.read(buf),
+ on_current_space=_UniffiFfiConverterOptionalBoolean.read(buf),
+ space_ids=_UniffiFfiConverterOptionalSequenceUInt64.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterUInt64.check_lower(value.window_id)
+ _UniffiFfiConverterOptionalUInt32.check_lower(value.pid)
+ _UniffiFfiConverterString.check_lower(value.app_name)
+ _UniffiFfiConverterString.check_lower(value.title)
+ _UniffiFfiConverterTypeWindowBounds.check_lower(value.bounds)
+ _UniffiFfiConverterBoolean.check_lower(value.is_on_screen)
+ _UniffiFfiConverterOptionalInt64.check_lower(value.z_index)
+ _UniffiFfiConverterOptionalInt32.check_lower(value.layer)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.minimized)
+ _UniffiFfiConverterOptionalUInt64.check_lower(value.current_space_id)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.on_current_space)
+ _UniffiFfiConverterOptionalSequenceUInt64.check_lower(value.space_ids)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterUInt64.write(value.window_id, buf)
+ _UniffiFfiConverterOptionalUInt32.write(value.pid, buf)
+ _UniffiFfiConverterString.write(value.app_name, buf)
+ _UniffiFfiConverterString.write(value.title, buf)
+ _UniffiFfiConverterTypeWindowBounds.write(value.bounds, buf)
+ _UniffiFfiConverterBoolean.write(value.is_on_screen, buf)
+ _UniffiFfiConverterOptionalInt64.write(value.z_index, buf)
+ _UniffiFfiConverterOptionalInt32.write(value.layer, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.minimized, buf)
+ _UniffiFfiConverterOptionalUInt64.write(value.current_space_id, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.on_current_space, buf)
+ _UniffiFfiConverterOptionalSequenceUInt64.write(value.space_ids, buf)
+
+class _UniffiFfiConverterSequenceTypeWindowInfo(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ for item in value:
+ _UniffiFfiConverterTypeWindowInfo.check_lower(item)
@classmethod
def write(cls, value, buf):
items = len(value)
buf.write_i32(items)
for item in value:
- _UniffiFfiConverterTypeSessionOutput.write(item, buf)
+ _UniffiFfiConverterTypeWindowInfo.write(item, buf)
@classmethod
def read(cls, buf):
@@ -3675,44 +4479,44 @@ class _UniffiFfiConverterSequenceTypeSessionOutput(_UniffiConverterRustBuffer):
raise InternalError("Unexpected negative sequence length")
return [
- _UniffiFfiConverterTypeSessionOutput.read(buf) for i in range(count)
+ _UniffiFfiConverterTypeWindowInfo.read(buf) for i in range(count)
]
@dataclass
-class ListSessionsOutput:
- def __init__(self, *, sessions:typing.List[SessionOutput], next_cursor:typing.Optional[str]):
- self.sessions = sessions
- self.next_cursor = next_cursor
+class ListWindowsOutput:
+ def __init__(self, *, windows:typing.List[WindowInfo], current_space_id:typing.Optional[int]):
+ self.windows = windows
+ self.current_space_id = current_space_id
def __str__(self):
- return "ListSessionsOutput(sessions={}, next_cursor={})".format(self.sessions, self.next_cursor)
+ return "ListWindowsOutput(windows={}, current_space_id={})".format(self.windows, self.current_space_id)
def __eq__(self, other):
- if self.sessions != other.sessions:
+ if self.windows != other.windows:
return False
- if self.next_cursor != other.next_cursor:
+ if self.current_space_id != other.current_space_id:
return False
return True
-class _UniffiFfiConverterTypeListSessionsOutput(_UniffiConverterRustBuffer):
+class _UniffiFfiConverterTypeListWindowsOutput(_UniffiConverterRustBuffer):
@staticmethod
def read(buf):
- return ListSessionsOutput(
- sessions=_UniffiFfiConverterSequenceTypeSessionOutput.read(buf),
- next_cursor=_UniffiFfiConverterOptionalString.read(buf),
+ return ListWindowsOutput(
+ windows=_UniffiFfiConverterSequenceTypeWindowInfo.read(buf),
+ current_space_id=_UniffiFfiConverterOptionalUInt64.read(buf),
)
@staticmethod
def check_lower(value):
- _UniffiFfiConverterSequenceTypeSessionOutput.check_lower(value.sessions)
- _UniffiFfiConverterOptionalString.check_lower(value.next_cursor)
+ _UniffiFfiConverterSequenceTypeWindowInfo.check_lower(value.windows)
+ _UniffiFfiConverterOptionalUInt64.check_lower(value.current_space_id)
@staticmethod
def write(value, buf):
- _UniffiFfiConverterSequenceTypeSessionOutput.write(value.sessions, buf)
- _UniffiFfiConverterOptionalString.write(value.next_cursor, buf)
+ _UniffiFfiConverterSequenceTypeWindowInfo.write(value.windows, buf)
+ _UniffiFfiConverterOptionalUInt64.write(value.current_space_id, buf)
@dataclass
class MoveCursorInput:
@@ -4734,6 +5538,42 @@ class _UniffiFfiConverterTypeSetWindowFrameInput(_UniffiConverterRustBuffer):
_UniffiFfiConverterFloat64.write(value.height, buf)
_UniffiFfiConverterOptionalString.write(value.session, buf)
+@dataclass
+class SnapshotImage:
+ def __init__(self, *, mime_type:str, data_base64:str):
+ self.mime_type = mime_type
+ self.data_base64 = data_base64
+
+
+
+
+ def __str__(self):
+ return "SnapshotImage(mime_type={}, data_base64={})".format(self.mime_type, self.data_base64)
+ def __eq__(self, other):
+ if self.mime_type != other.mime_type:
+ return False
+ if self.data_base64 != other.data_base64:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeSnapshotImage(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return SnapshotImage(
+ mime_type=_UniffiFfiConverterString.read(buf),
+ data_base64=_UniffiFfiConverterString.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterString.check_lower(value.mime_type)
+ _UniffiFfiConverterString.check_lower(value.data_base64)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterString.write(value.mime_type, buf)
+ _UniffiFfiConverterString.write(value.data_base64, buf)
+
class _UniffiFfiConverterOptionalTypeCaptureScope(_UniffiConverterRustBuffer):
@classmethod
def check_lower(cls, value):
@@ -5066,19 +5906,6 @@ class _UniffiFfiConverterTypeTypeTextInput(_UniffiConverterRustBuffer):
_UniffiFfiConverterOptionalTypeDesktopScope.write(value.scope, buf)
_UniffiFfiConverterOptionalString.write(value.session, buf)
-class _UniffiFfiConverterInt64(_UniffiConverterPrimitiveInt):
- CLASS_NAME = "i64"
- VALUE_MIN = -2**63
- VALUE_MAX = 2**63
-
- @staticmethod
- def read(buf):
- return buf.read_i64()
-
- @staticmethod
- def write(value, buf):
- buf.write_i64(value)
-
class _UniffiFfiConverterSequenceTypeStatePredicate(_UniffiConverterRustBuffer):
@classmethod
def check_lower(cls, value):
@@ -5245,6 +6072,409 @@ class _UniffiFfiConverterTypeVerifyStateOutput(_UniffiConverterRustBuffer):
_UniffiFfiConverterUInt64.write(value.samples, buf)
_UniffiFfiConverterSequenceTypePredicateOutcome.write(value.predicates, buf)
+class _UniffiFfiConverterOptionalTypeElementFrame(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterTypeElementFrame.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterTypeElementFrame.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterTypeElementFrame.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
+@dataclass
+class WindowElement:
+ def __init__(self, *, element_index:int, role:str, depth:int, element_token:typing.Optional[str], label:typing.Optional[str], value:typing.Optional[str], value_description:typing.Optional[str], enabled:typing.Optional[bool], selected:typing.Optional[bool], in_web_content:typing.Optional[bool], actions:typing.Optional[typing.List[str]], parent_index:typing.Optional[int], frame:typing.Optional[ElementFrame], min:typing.Optional[float], max:typing.Optional[float]):
+ self.element_index = element_index
+ self.role = role
+ self.depth = depth
+ self.element_token = element_token
+ self.label = label
+ self.value = value
+ self.value_description = value_description
+ self.enabled = enabled
+ self.selected = selected
+ self.in_web_content = in_web_content
+ self.actions = actions
+ self.parent_index = parent_index
+ self.frame = frame
+ self.min = min
+ self.max = max
+
+
+
+
+ def __str__(self):
+ return "WindowElement(element_index={}, role={}, depth={}, element_token={}, label={}, value={}, value_description={}, enabled={}, selected={}, in_web_content={}, actions={}, parent_index={}, frame={}, min={}, max={})".format(self.element_index, self.role, self.depth, self.element_token, self.label, self.value, self.value_description, self.enabled, self.selected, self.in_web_content, self.actions, self.parent_index, self.frame, self.min, self.max)
+ def __eq__(self, other):
+ if self.element_index != other.element_index:
+ return False
+ if self.role != other.role:
+ return False
+ if self.depth != other.depth:
+ return False
+ if self.element_token != other.element_token:
+ return False
+ if self.label != other.label:
+ return False
+ if self.value != other.value:
+ return False
+ if self.value_description != other.value_description:
+ return False
+ if self.enabled != other.enabled:
+ return False
+ if self.selected != other.selected:
+ return False
+ if self.in_web_content != other.in_web_content:
+ return False
+ if self.actions != other.actions:
+ return False
+ if self.parent_index != other.parent_index:
+ return False
+ if self.frame != other.frame:
+ return False
+ if self.min != other.min:
+ return False
+ if self.max != other.max:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeWindowElement(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return WindowElement(
+ element_index=_UniffiFfiConverterUInt64.read(buf),
+ role=_UniffiFfiConverterString.read(buf),
+ depth=_UniffiFfiConverterUInt32.read(buf),
+ element_token=_UniffiFfiConverterOptionalString.read(buf),
+ label=_UniffiFfiConverterOptionalString.read(buf),
+ value=_UniffiFfiConverterOptionalString.read(buf),
+ value_description=_UniffiFfiConverterOptionalString.read(buf),
+ enabled=_UniffiFfiConverterOptionalBoolean.read(buf),
+ selected=_UniffiFfiConverterOptionalBoolean.read(buf),
+ in_web_content=_UniffiFfiConverterOptionalBoolean.read(buf),
+ actions=_UniffiFfiConverterOptionalSequenceString.read(buf),
+ parent_index=_UniffiFfiConverterOptionalUInt64.read(buf),
+ frame=_UniffiFfiConverterOptionalTypeElementFrame.read(buf),
+ min=_UniffiFfiConverterOptionalFloat64.read(buf),
+ max=_UniffiFfiConverterOptionalFloat64.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterUInt64.check_lower(value.element_index)
+ _UniffiFfiConverterString.check_lower(value.role)
+ _UniffiFfiConverterUInt32.check_lower(value.depth)
+ _UniffiFfiConverterOptionalString.check_lower(value.element_token)
+ _UniffiFfiConverterOptionalString.check_lower(value.label)
+ _UniffiFfiConverterOptionalString.check_lower(value.value)
+ _UniffiFfiConverterOptionalString.check_lower(value.value_description)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.enabled)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.selected)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.in_web_content)
+ _UniffiFfiConverterOptionalSequenceString.check_lower(value.actions)
+ _UniffiFfiConverterOptionalUInt64.check_lower(value.parent_index)
+ _UniffiFfiConverterOptionalTypeElementFrame.check_lower(value.frame)
+ _UniffiFfiConverterOptionalFloat64.check_lower(value.min)
+ _UniffiFfiConverterOptionalFloat64.check_lower(value.max)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterUInt64.write(value.element_index, buf)
+ _UniffiFfiConverterString.write(value.role, buf)
+ _UniffiFfiConverterUInt32.write(value.depth, buf)
+ _UniffiFfiConverterOptionalString.write(value.element_token, buf)
+ _UniffiFfiConverterOptionalString.write(value.label, buf)
+ _UniffiFfiConverterOptionalString.write(value.value, buf)
+ _UniffiFfiConverterOptionalString.write(value.value_description, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.enabled, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.selected, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.in_web_content, buf)
+ _UniffiFfiConverterOptionalSequenceString.write(value.actions, buf)
+ _UniffiFfiConverterOptionalUInt64.write(value.parent_index, buf)
+ _UniffiFfiConverterOptionalTypeElementFrame.write(value.frame, buf)
+ _UniffiFfiConverterOptionalFloat64.write(value.min, buf)
+ _UniffiFfiConverterOptionalFloat64.write(value.max, buf)
+
+class _UniffiFfiConverterSequenceTypeWindowElement(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ for item in value:
+ _UniffiFfiConverterTypeWindowElement.check_lower(item)
+
+ @classmethod
+ def write(cls, value, buf):
+ items = len(value)
+ buf.write_i32(items)
+ for item in value:
+ _UniffiFfiConverterTypeWindowElement.write(item, buf)
+
+ @classmethod
+ def read(cls, buf):
+ count = buf.read_i32()
+ if count < 0:
+ raise InternalError("Unexpected negative sequence length")
+
+ return [
+ _UniffiFfiConverterTypeWindowElement.read(buf) for i in range(count)
+ ]
+
+class _UniffiFfiConverterOptionalSequenceTypeWindowElement(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterSequenceTypeWindowElement.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterSequenceTypeWindowElement.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterSequenceTypeWindowElement.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
+class _UniffiFfiConverterOptionalTypeWindowBounds(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ if value is not None:
+ _UniffiFfiConverterTypeWindowBounds.check_lower(value)
+
+ @classmethod
+ def write(cls, value, buf):
+ if value is None:
+ buf.write_u8(0)
+ return
+
+ buf.write_u8(1)
+ _UniffiFfiConverterTypeWindowBounds.write(value, buf)
+
+ @classmethod
+ def read(cls, buf):
+ flag = buf.read_u8()
+ if flag == 0:
+ return None
+ elif flag == 1:
+ return _UniffiFfiConverterTypeWindowBounds.read(buf)
+ else:
+ raise InternalError("Unexpected flag byte for optional type")
+
+class _UniffiFfiConverterSequenceTypeSnapshotImage(_UniffiConverterRustBuffer):
+ @classmethod
+ def check_lower(cls, value):
+ for item in value:
+ _UniffiFfiConverterTypeSnapshotImage.check_lower(item)
+
+ @classmethod
+ def write(cls, value, buf):
+ items = len(value)
+ buf.write_i32(items)
+ for item in value:
+ _UniffiFfiConverterTypeSnapshotImage.write(item, buf)
+
+ @classmethod
+ def read(cls, buf):
+ count = buf.read_i32()
+ if count < 0:
+ raise InternalError("Unexpected negative sequence length")
+
+ return [
+ _UniffiFfiConverterTypeSnapshotImage.read(buf) for i in range(count)
+ ]
+
+@dataclass
+class WindowStateOutput:
+ def __init__(self, *, pid:int, window_id:int, snapshot_id:typing.Optional[str], app_name:typing.Optional[str], window_title:typing.Optional[str], tree_markdown:typing.Optional[str], elements:typing.Optional[typing.List[WindowElement]], element_count:typing.Optional[int], total_element_count:typing.Optional[int], returned_element_count:typing.Optional[int], filtered_element_count:typing.Optional[int], elements_complete:typing.Optional[bool], degraded:typing.Optional[bool], degraded_reason:typing.Optional[str], truncated:typing.Optional[bool], truncation_reason:typing.Optional[str], screenshot_width:typing.Optional[int], screenshot_height:typing.Optional[int], screenshot_scale:typing.Optional[float], screenshot_mime_type:typing.Optional[str], screenshot_file_path:typing.Optional[str], screenshot_frame_valid:typing.Optional[bool], window_bounds:typing.Optional[WindowBounds], images:typing.List[SnapshotImage]):
+ self.pid = pid
+ self.window_id = window_id
+ self.snapshot_id = snapshot_id
+ self.app_name = app_name
+ self.window_title = window_title
+ self.tree_markdown = tree_markdown
+ self.elements = elements
+ self.element_count = element_count
+ self.total_element_count = total_element_count
+ self.returned_element_count = returned_element_count
+ self.filtered_element_count = filtered_element_count
+ self.elements_complete = elements_complete
+ self.degraded = degraded
+ self.degraded_reason = degraded_reason
+ self.truncated = truncated
+ self.truncation_reason = truncation_reason
+ self.screenshot_width = screenshot_width
+ self.screenshot_height = screenshot_height
+ self.screenshot_scale = screenshot_scale
+ self.screenshot_mime_type = screenshot_mime_type
+ self.screenshot_file_path = screenshot_file_path
+ self.screenshot_frame_valid = screenshot_frame_valid
+ self.window_bounds = window_bounds
+ self.images = images
+
+
+
+
+ def __str__(self):
+ return "WindowStateOutput(pid={}, window_id={}, snapshot_id={}, app_name={}, window_title={}, tree_markdown={}, elements={}, element_count={}, total_element_count={}, returned_element_count={}, filtered_element_count={}, elements_complete={}, degraded={}, degraded_reason={}, truncated={}, truncation_reason={}, screenshot_width={}, screenshot_height={}, screenshot_scale={}, screenshot_mime_type={}, screenshot_file_path={}, screenshot_frame_valid={}, window_bounds={}, images={})".format(self.pid, self.window_id, self.snapshot_id, self.app_name, self.window_title, self.tree_markdown, self.elements, self.element_count, self.total_element_count, self.returned_element_count, self.filtered_element_count, self.elements_complete, self.degraded, self.degraded_reason, self.truncated, self.truncation_reason, self.screenshot_width, self.screenshot_height, self.screenshot_scale, self.screenshot_mime_type, self.screenshot_file_path, self.screenshot_frame_valid, self.window_bounds, self.images)
+ def __eq__(self, other):
+ if self.pid != other.pid:
+ return False
+ if self.window_id != other.window_id:
+ return False
+ if self.snapshot_id != other.snapshot_id:
+ return False
+ if self.app_name != other.app_name:
+ return False
+ if self.window_title != other.window_title:
+ return False
+ if self.tree_markdown != other.tree_markdown:
+ return False
+ if self.elements != other.elements:
+ return False
+ if self.element_count != other.element_count:
+ return False
+ if self.total_element_count != other.total_element_count:
+ return False
+ if self.returned_element_count != other.returned_element_count:
+ return False
+ if self.filtered_element_count != other.filtered_element_count:
+ return False
+ if self.elements_complete != other.elements_complete:
+ return False
+ if self.degraded != other.degraded:
+ return False
+ if self.degraded_reason != other.degraded_reason:
+ return False
+ if self.truncated != other.truncated:
+ return False
+ if self.truncation_reason != other.truncation_reason:
+ return False
+ if self.screenshot_width != other.screenshot_width:
+ return False
+ if self.screenshot_height != other.screenshot_height:
+ return False
+ if self.screenshot_scale != other.screenshot_scale:
+ return False
+ if self.screenshot_mime_type != other.screenshot_mime_type:
+ return False
+ if self.screenshot_file_path != other.screenshot_file_path:
+ return False
+ if self.screenshot_frame_valid != other.screenshot_frame_valid:
+ return False
+ if self.window_bounds != other.window_bounds:
+ return False
+ if self.images != other.images:
+ return False
+ return True
+
+class _UniffiFfiConverterTypeWindowStateOutput(_UniffiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ return WindowStateOutput(
+ pid=_UniffiFfiConverterUInt32.read(buf),
+ window_id=_UniffiFfiConverterUInt64.read(buf),
+ snapshot_id=_UniffiFfiConverterOptionalString.read(buf),
+ app_name=_UniffiFfiConverterOptionalString.read(buf),
+ window_title=_UniffiFfiConverterOptionalString.read(buf),
+ tree_markdown=_UniffiFfiConverterOptionalString.read(buf),
+ elements=_UniffiFfiConverterOptionalSequenceTypeWindowElement.read(buf),
+ element_count=_UniffiFfiConverterOptionalUInt64.read(buf),
+ total_element_count=_UniffiFfiConverterOptionalUInt64.read(buf),
+ returned_element_count=_UniffiFfiConverterOptionalUInt64.read(buf),
+ filtered_element_count=_UniffiFfiConverterOptionalUInt64.read(buf),
+ elements_complete=_UniffiFfiConverterOptionalBoolean.read(buf),
+ degraded=_UniffiFfiConverterOptionalBoolean.read(buf),
+ degraded_reason=_UniffiFfiConverterOptionalString.read(buf),
+ truncated=_UniffiFfiConverterOptionalBoolean.read(buf),
+ truncation_reason=_UniffiFfiConverterOptionalString.read(buf),
+ screenshot_width=_UniffiFfiConverterOptionalUInt32.read(buf),
+ screenshot_height=_UniffiFfiConverterOptionalUInt32.read(buf),
+ screenshot_scale=_UniffiFfiConverterOptionalFloat64.read(buf),
+ screenshot_mime_type=_UniffiFfiConverterOptionalString.read(buf),
+ screenshot_file_path=_UniffiFfiConverterOptionalString.read(buf),
+ screenshot_frame_valid=_UniffiFfiConverterOptionalBoolean.read(buf),
+ window_bounds=_UniffiFfiConverterOptionalTypeWindowBounds.read(buf),
+ images=_UniffiFfiConverterSequenceTypeSnapshotImage.read(buf),
+ )
+
+ @staticmethod
+ def check_lower(value):
+ _UniffiFfiConverterUInt32.check_lower(value.pid)
+ _UniffiFfiConverterUInt64.check_lower(value.window_id)
+ _UniffiFfiConverterOptionalString.check_lower(value.snapshot_id)
+ _UniffiFfiConverterOptionalString.check_lower(value.app_name)
+ _UniffiFfiConverterOptionalString.check_lower(value.window_title)
+ _UniffiFfiConverterOptionalString.check_lower(value.tree_markdown)
+ _UniffiFfiConverterOptionalSequenceTypeWindowElement.check_lower(value.elements)
+ _UniffiFfiConverterOptionalUInt64.check_lower(value.element_count)
+ _UniffiFfiConverterOptionalUInt64.check_lower(value.total_element_count)
+ _UniffiFfiConverterOptionalUInt64.check_lower(value.returned_element_count)
+ _UniffiFfiConverterOptionalUInt64.check_lower(value.filtered_element_count)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.elements_complete)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.degraded)
+ _UniffiFfiConverterOptionalString.check_lower(value.degraded_reason)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.truncated)
+ _UniffiFfiConverterOptionalString.check_lower(value.truncation_reason)
+ _UniffiFfiConverterOptionalUInt32.check_lower(value.screenshot_width)
+ _UniffiFfiConverterOptionalUInt32.check_lower(value.screenshot_height)
+ _UniffiFfiConverterOptionalFloat64.check_lower(value.screenshot_scale)
+ _UniffiFfiConverterOptionalString.check_lower(value.screenshot_mime_type)
+ _UniffiFfiConverterOptionalString.check_lower(value.screenshot_file_path)
+ _UniffiFfiConverterOptionalBoolean.check_lower(value.screenshot_frame_valid)
+ _UniffiFfiConverterOptionalTypeWindowBounds.check_lower(value.window_bounds)
+ _UniffiFfiConverterSequenceTypeSnapshotImage.check_lower(value.images)
+
+ @staticmethod
+ def write(value, buf):
+ _UniffiFfiConverterUInt32.write(value.pid, buf)
+ _UniffiFfiConverterUInt64.write(value.window_id, buf)
+ _UniffiFfiConverterOptionalString.write(value.snapshot_id, buf)
+ _UniffiFfiConverterOptionalString.write(value.app_name, buf)
+ _UniffiFfiConverterOptionalString.write(value.window_title, buf)
+ _UniffiFfiConverterOptionalString.write(value.tree_markdown, buf)
+ _UniffiFfiConverterOptionalSequenceTypeWindowElement.write(value.elements, buf)
+ _UniffiFfiConverterOptionalUInt64.write(value.element_count, buf)
+ _UniffiFfiConverterOptionalUInt64.write(value.total_element_count, buf)
+ _UniffiFfiConverterOptionalUInt64.write(value.returned_element_count, buf)
+ _UniffiFfiConverterOptionalUInt64.write(value.filtered_element_count, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.elements_complete, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.degraded, buf)
+ _UniffiFfiConverterOptionalString.write(value.degraded_reason, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.truncated, buf)
+ _UniffiFfiConverterOptionalString.write(value.truncation_reason, buf)
+ _UniffiFfiConverterOptionalUInt32.write(value.screenshot_width, buf)
+ _UniffiFfiConverterOptionalUInt32.write(value.screenshot_height, buf)
+ _UniffiFfiConverterOptionalFloat64.write(value.screenshot_scale, buf)
+ _UniffiFfiConverterOptionalString.write(value.screenshot_mime_type, buf)
+ _UniffiFfiConverterOptionalString.write(value.screenshot_file_path, buf)
+ _UniffiFfiConverterOptionalBoolean.write(value.screenshot_frame_valid, buf)
+ _UniffiFfiConverterOptionalTypeWindowBounds.write(value.window_bounds, buf)
+ _UniffiFfiConverterSequenceTypeSnapshotImage.write(value.images, buf)
+
@@ -5315,10 +6545,12 @@ __all__ = [
"ActionEffect",
"ActionRoute",
"ActionTarget",
- "DesktopScope",
+ "ClickPosition",
+ "InputDeliveryMode",
"ClickButton",
"CursorReducedMotion",
"CursorAction",
+ "DesktopScope",
"EscalationReason",
"SessionLifecycleState",
"SessionClientKindOutput",
@@ -5334,6 +6566,7 @@ __all__ = [
"ActionEscalation",
"ActionEvidence",
"ActionResult",
+ "AppInfo",
"BoundsExpectation",
"ClickInput",
"ClipboardReadInput",
@@ -5346,6 +6579,7 @@ __all__ = [
"CursorThemeSelection",
"CursorVisualOutput",
"DragInput",
+ "ElementFrame",
"ElementSelector",
"ElementPredicate",
"EndSessionInput",
@@ -5358,11 +6592,18 @@ __all__ = [
"GetScreenSizeInput",
"GetSessionInput",
"GetSessionStateInput",
+ "GetWindowStateInput",
"HotkeyInput",
"InvokeMenuInput",
+ "ListAppsInput",
+ "ListAppsOutput",
"ListSessionsInput",
"SessionOutput",
"ListSessionsOutput",
+ "ListWindowsInput",
+ "WindowBounds",
+ "WindowInfo",
+ "ListWindowsOutput",
"MoveCursorInput",
"PredicateOutcome",
"PressKeyInput",
@@ -5375,6 +6616,7 @@ __all__ = [
"SetAgentCursorThemeInput",
"SetAgentCursorThemeOutput",
"SetWindowFrameInput",
+ "SnapshotImage",
"StartSessionInput",
"StartSessionOutput",
"WindowPredicate",
@@ -5382,4 +6624,6 @@ __all__ = [
"TypeTextInput",
"VerifyStateInput",
"VerifyStateOutput",
+ "WindowElement",
+ "WindowStateOutput",
]
diff --git a/libs/cua-driver/python/tests/test_compatibility_contract.py b/libs/cua-driver/python/tests/test_compatibility_contract.py
index 70e763a09d..6c77fcc382 100644
--- a/libs/cua-driver/python/tests/test_compatibility_contract.py
+++ b/libs/cua-driver/python/tests/test_compatibility_contract.py
@@ -61,6 +61,24 @@ def _functions(module: ast.Module) -> dict[str, ast.FunctionDef | ast.AsyncFunct
}
+def test_current_native_window_methods_have_typed_outputs() -> None:
+ module = _module(PACKAGE_ROOT / "src" / "cua_driver" / "_native.py")
+ driver = next(
+ node for node in module.body if isinstance(node, ast.ClassDef) and node.name == "CuaDriver"
+ )
+ methods = _functions(driver)
+ for name, input_type, output_type in [
+ ("list_apps", "ListAppsInput", "ListAppsOutput"),
+ ("list_windows", "ListWindowsInput", "ListWindowsOutput"),
+ ("get_window_state", "GetWindowStateInput", "WindowStateOutput"),
+ ("click", "ClickInput", "ActionResult"),
+ ]:
+ assert _signature(methods[name]) == (
+ f"async {name}(self, input: cua_driver._native_contract.{input_type})"
+ f" -> cua_driver._native_contract.{output_type}"
+ )
+
+
def test_released_python_exports_and_signatures_remain_available() -> None:
expected = json.loads(FIXTURE.read_text(encoding="utf-8"))
package_module = _module(PACKAGE_ROOT / "src" / "cua_driver" / "__init__.py")
diff --git a/libs/cua-driver/python/tests/test_remote_channel.py b/libs/cua-driver/python/tests/test_remote_channel.py
index ad59a4cd26..de17c598ee 100644
--- a/libs/cua-driver/python/tests/test_remote_channel.py
+++ b/libs/cua-driver/python/tests/test_remote_channel.py
@@ -35,6 +35,7 @@ class Carrier(ForeignDriverEnvelopeChannel):
self.cancelled_id = None
self.bound = None
self.response_mode = "ok"
+ self.result = None
self.supports_cancellation = True
self.identity_error = False
self.minimum_version = 1
@@ -64,7 +65,7 @@ class Carrier(ForeignDriverEnvelopeChannel):
await asyncio.Event().wait()
if self.response_mode == "error":
raise ForeignDriverChannelError.Failed("fixture disconnected")
- result = {
+ result = self.result if self.result is not None else {
"content": [{"type": "text", "text": "carrier response"}],
"isError": False,
}
@@ -244,5 +245,153 @@ class RemoteChannelTests(unittest.IsolatedAsyncioTestCase):
await driver.shutdown()
+class NativeWindowTests(unittest.IsolatedAsyncioTestCase):
+ async def test_typed_discovery_snapshot_and_token_click_cross_ffi(self):
+ import cua_driver as sdk
+
+ carrier = Carrier()
+ driver = connect_remote_channel(carrier)
+
+ def respond(structured, *, images=False, error=False):
+ carrier.result = {
+ "content": [{"type": "text", "text": "native window fixture"}],
+ "structuredContent": structured,
+ "isError": error,
+ }
+ if images:
+ carrier.result["content"].append(
+ {"type": "image", "mimeType": "image/png", "data": "cG5n"}
+ )
+
+ try:
+ respond({"apps": [{"pid": 42, "name": "Editor", "running": True, "active": False}]})
+ apps = await driver.list_apps(sdk.ListAppsInput())
+ self.assertIsInstance(apps, sdk.ListAppsOutput)
+ self.assertEqual(apps.apps[0].name, "Editor")
+ self.assertEqual(json.loads(carrier.requests[-1].arguments_json), {})
+
+ respond(
+ {
+ "windows": [
+ {
+ "pid": 42,
+ "window_id": 123,
+ "app_name": "Editor",
+ "title": "Document",
+ "bounds": {"x": 0, "y": 0, "width": 800, "height": 600},
+ "is_on_screen": True,
+ "z_index": None,
+ }
+ ]
+ }
+ )
+ windows = await driver.list_windows(sdk.ListWindowsInput(pid=42, on_screen_only=True))
+ self.assertIsInstance(windows.windows[0], sdk.WindowInfo)
+ self.assertEqual(windows.windows[0].bounds.width, 800)
+ self.assertIsNone(windows.windows[0].z_index)
+ self.assertEqual(
+ json.loads(carrier.requests[-1].arguments_json), {"pid": 42, "on_screen_only": True}
+ )
+
+ respond(
+ {
+ "pid": 42,
+ "window_id": 123,
+ "snapshot_id": "snapshot-1",
+ "screenshot_width": 800,
+ "screenshot_height": 600,
+ "elements": [
+ {
+ "element_index": 0,
+ "role": "button",
+ "depth": 0,
+ "element_token": "fresh-token",
+ "label": "Save",
+ }
+ ],
+ },
+ images=True,
+ )
+ state = await driver.get_window_state(
+ sdk.GetWindowStateInput(
+ pid=42,
+ window_id=123,
+ session="native-window",
+ query="Save",
+ include_screenshot=True,
+ include_accessibility_tree=True,
+ screenshot_out_file=None,
+ max_elements=10,
+ max_depth=3,
+ max_dimension=800,
+ )
+ )
+ self.assertIsInstance(state, sdk.WindowStateOutput)
+ self.assertIsInstance(state.elements[0], sdk.WindowElement)
+ self.assertEqual(
+ state.images[0], sdk.SnapshotImage(mime_type="image/png", data_base64="cG5n")
+ )
+ self.assertEqual(
+ json.loads(carrier.requests[-1].arguments_json),
+ {
+ "pid": 42,
+ "window_id": 123,
+ "session": "native-window",
+ "query": "Save",
+ "include_screenshot": True,
+ "include_accessibility_tree": True,
+ "max_elements": 10,
+ "max_depth": 3,
+ "max_dimension": 800,
+ },
+ )
+
+ respond(
+ {
+ "effect": "unverifiable",
+ "route": "global_input",
+ "delivery": {"mode": "not_applicable"},
+ }
+ )
+ click = sdk.ClickInput(
+ target=sdk.ActionTarget.WINDOW(pid=42, window_id=123),
+ position=sdk.ClickPosition.ELEMENT(element_token=state.elements[0].element_token),
+ delivery_mode=sdk.InputDeliveryMode.BACKGROUND,
+ session="native-window",
+ button=None,
+ count=None,
+ )
+ action = await driver.click(click)
+ self.assertIsInstance(action, sdk.ActionResult)
+ self.assertEqual(action.effect, sdk.ActionEffect.UNVERIFIABLE)
+ self.assertEqual(
+ json.loads(carrier.requests[-1].arguments_json),
+ {
+ "target": {"kind": "window", "pid": 42, "window_id": 123},
+ "element_token": "fresh-token",
+ "delivery_mode": "background",
+ "session": "native-window",
+ },
+ )
+
+ # These service refusals test SDK error propagation, not native token validation.
+ for token, window_id, code in [
+ ("stale-token", 123, "stale_element_token"),
+ ("fresh-token", 124, "element_target_mismatch"),
+ ]:
+ respond({"code": code}, error=True)
+ click.position = sdk.ClickPosition.ELEMENT(element_token=token)
+ click.target = sdk.ActionTarget.WINDOW(pid=42, window_id=window_id)
+ with self.assertRaises(sdk.DriverError.Tool) as error:
+ await driver.click(click)
+ self.assertEqual(error.exception.tool, "click")
+ self.assertEqual(error.exception.error_code, code)
+ request = json.loads(carrier.requests[-1].arguments_json)
+ self.assertEqual(request["element_token"], token)
+ self.assertEqual(request["target"]["window_id"], window_id)
+ finally:
+ await driver.shutdown()
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/libs/cua-driver/python/tests/test_uniffi_loader.py b/libs/cua-driver/python/tests/test_uniffi_loader.py
index d4e70b07eb..344b2aeb04 100644
--- a/libs/cua-driver/python/tests/test_uniffi_loader.py
+++ b/libs/cua-driver/python/tests/test_uniffi_loader.py
@@ -27,18 +27,31 @@ LIBRARY = Path(__file__).parents[1] / "src" / "cua_driver" / _library_name()
class GeneratedOptionsTests(unittest.TestCase):
def test_action_target_is_public_and_uses_the_generated_contract(self) -> None:
import cua_driver
- from cua_driver import ActionTarget, ClickButton, ClickInput
+ from cua_driver import (
+ ActionTarget, ClickButton, ClickInput, ClickPosition, InputDeliveryMode,
+ )
from cua_driver._native_contract import ActionTarget as GeneratedActionTarget
self.assertIn("ActionTarget", cua_driver.__all__)
self.assertIs(ActionTarget, GeneratedActionTarget)
target = ActionTarget.DESKTOP(display_id="primary")
click = ClickInput(
- x=10.0, y=20.0, target=target, scope=None, session=None,
+ position=ClickPosition.COORDINATES(x=10.0, y=20.0), target=target,
+ delivery_mode=InputDeliveryMode.FOREGROUND, session=None,
button=ClickButton.LEFT, count=1,
)
self.assertIs(click.target, target)
self.assertTrue(ActionTarget.WINDOW(pid=1, window_id=2).is_WINDOW())
+ with self.assertRaises(TypeError):
+ ClickInput(x=10.0, y=20.0)
+ for name in (
+ "AppInfo", "ClickPosition", "ElementFrame", "GetWindowStateInput",
+ "InputDeliveryMode", "ListAppsInput", "ListAppsOutput", "ListWindowsInput",
+ "ListWindowsOutput", "SnapshotImage", "WindowBounds", "WindowElement",
+ "WindowInfo", "WindowStateOutput",
+ ):
+ self.assertIn(name, cua_driver.__all__)
+ self.assertIsNotNone(getattr(cua_driver, name))
def test_embedded_overlay_option_defaults_false_and_accepts_true(self) -> None:
from cua_driver import EmbeddedDriverHostOptions
@@ -98,7 +111,7 @@ while True:
if request["method"] == "metadata":
result = {
"driver_version": "0.10.0",
- "contract_version": "0.7.0",
+ "contract_version": "0.8.0",
"tools_list_schema_version": "1",
"capability_version": "1",
"mcp_protocol_version": "2025-06-18",
@@ -146,10 +159,12 @@ except FileNotFoundError:
from cua_driver import (
ActionEffect,
ActionRoute,
+ ActionTarget,
+ ClickPosition,
+ InputDeliveryMode,
ClickButton,
ClickInput,
CuaDriver,
- DesktopScope,
EffectiveScope,
StatePredicate,
StartSessionOutput,
@@ -182,7 +197,7 @@ except FileNotFoundError:
if request["method"] == "metadata":
result = {
"driver_version": "0.12.6",
- "contract_version": "0.7.0",
+ "contract_version": "0.8.0",
"tools_list_schema_version": "1",
"capability_version": "1",
"mcp_protocol_version": "2025-06-18",
@@ -264,10 +279,9 @@ except FileNotFoundError:
action_result = asyncio.run(
driver.click(
ClickInput(
- x=12.0,
- y=34.0,
- target=None,
- scope=DesktopScope.DESKTOP,
+ position=ClickPosition.COORDINATES(x=12.0, y=34.0),
+ target=ActionTarget.DESKTOP(display_id="primary"),
+ delivery_mode=InputDeliveryMode.FOREGROUND,
session="python-run",
button=ClickButton.LEFT,
count=1,
@@ -283,9 +297,8 @@ except FileNotFoundError:
self.assertEqual(
verification_result.verification.status, VerificationStatus.SATISFIED
)
- self.assertIsNone(action_result.verification)
- self.assertEqual(action_result.action.effect, ActionEffect.UNVERIFIABLE)
- self.assertEqual(action_result.action.route, ActionRoute.GLOBAL_INPUT)
+ self.assertEqual(action_result.effect, ActionEffect.UNVERIFIABLE)
+ self.assertEqual(action_result.route, ActionRoute.GLOBAL_INPUT)
self.assertFalse(hasattr(action_result, "verified"))
self.assertEqual(captured[0]["name"], "verify_state")
self.assertEqual(
@@ -307,7 +320,8 @@ except FileNotFoundError:
{
"x": 12.0,
"y": 34.0,
- "scope": "desktop",
+ "target": {"kind": "desktop", "display_id": "primary"},
+ "delivery_mode": "foreground",
"session": "python-run",
"button": "left",
"count": 1,
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs
index 2471f3a178..e1f89013d4 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs
@@ -60,7 +60,18 @@ fn compare_schema(path: &str, portable: &Value, live: &Value, violations: &mut V
return;
};
- check_keywords(path, "portable", portable_object.keys(), violations);
+ // A proof for the base schema also proves the subset for a conjunction
+ // with extra union constraints. Ignore only portable unions here: the
+ // base must independently imply every live constraint. Never discard a
+ // live union, which would broaden the schema we are proving against.
+ check_keywords(
+ path,
+ "portable",
+ portable_object
+ .keys()
+ .filter(|key| !matches!(key.as_str(), "anyOf" | "oneOf")),
+ violations,
+ );
check_keywords(path, "live", live_object.keys(), violations);
compare_type(path, portable, live, violations);
@@ -312,6 +323,23 @@ mod tests {
use super::*;
+ #[test]
+ fn portable_union_is_safe_only_when_base_schema_proves_subset() {
+ let portable = json!({"type":"object","properties":{"x":{"type":"number"}},"additionalProperties":false,
+ "anyOf":[{"required":["x"]}]});
+ let live = json!({"type":"object","properties":{"x":{"type":"number"}},"additionalProperties":false});
+ assert!(schema_subset_violations(&portable, &live).is_empty());
+ let mut requiring = live.clone();
+ requiring["required"] = json!(["x"]);
+ assert!(!schema_subset_violations(&portable, &requiring).is_empty());
+ assert!(!schema_subset_violations(&live, &portable).is_empty());
+ assert!(!schema_subset_violations(
+ &json!({"anyOf":[{"type":"number"}]}),
+ &json!({"type":"number"})
+ )
+ .is_empty());
+ }
+
#[test]
fn narrower_portable_object_is_accepted() {
let portable = json!({
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs
index 93a3406591..f30c11709a 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs
@@ -10,16 +10,20 @@
use crate::{
ActionResult, ClickInput, ClipboardReadInput, ClipboardReadOutput, ClipboardWriteInput,
ClipboardWriteOutput, CursorAction, CursorPositionOutput, CursorSemantics, DesktopStateOutput,
- DragInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput, HotkeyInput,
- InvokeMenuInput, MoveCursorInput, Platform, PressKeyInput, SchemaMode, ScreenSizeOutput,
- ScrollInput, SetWindowFrameInput, ToolAnnotations, ToolContract, ToolInput, ToolOutput,
- TypeTextInput,
+ DragInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput,
+ GetWindowStateInput, HotkeyInput, InvokeMenuInput, ListAppsInput, ListAppsOutput,
+ ListWindowsInput, ListWindowsOutput, MoveCursorInput, Platform, PressKeyInput, SchemaMode,
+ ScreenSizeOutput, ScrollInput, SetWindowFrameInput, ToolAnnotations, ToolContract, ToolInput,
+ ToolOutput, TypeTextInput, WindowStateOutput,
};
const ALL_PLATFORMS: [Platform; 3] = [Platform::Macos, Platform::Windows, Platform::Linux];
pub fn contracts() -> Vec<ToolContract> {
vec![
+ list_apps(),
+ list_windows(),
+ get_window_state(),
get_desktop_state(),
get_screen_size(),
get_cursor_position(),
@@ -37,52 +41,57 @@ pub fn contracts() -> Vec<ToolContract> {
]
}
-const Z_INDEX_DESCRIPTION: &str = "Higher values are closer to the front. Null means the provider cannot observe stacking order; callers must not infer an order from array position or treat null as zero.";
+fn list_apps() -> ToolContract {
+ contract::<ListAppsInput, ListAppsOutput>(
+ "list_apps",
+ "Discover installed and running native applications.",
+ &["app.list"],
+ ToolAnnotations {
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
+ },
+ CursorAction::Observe,
+ )
+}
-// Keep this schema deliberately narrow: platform window records have additive
-// fields and are still converging, while z_index has one portable meaning that
-// consumers need in order to sort safely. This runtime schema intentionally
-// stays outside the typed SDK manifest until that broader shape converges.
-pub(crate) fn list_windows_success_output_schema() -> serde_json::Value {
- serde_json::json!({
- "type": "object",
- "properties": {
- "windows": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "z_index": {
- "type": ["integer", "null"],
- "description": Z_INDEX_DESCRIPTION
- }
- },
- "required": ["z_index"],
- "additionalProperties": true
- }
- }
- },
- "required": ["windows"],
- "additionalProperties": true
- })
+fn list_windows() -> ToolContract {
+ contract::<ListWindowsInput, ListWindowsOutput>(
+ "list_windows",
+ "Discover exact native windows and observable bounds and stacking order.",
+ &["window.list"],
+ ToolAnnotations {
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
+ },
+ CursorAction::Observe,
+ )
}
-pub(crate) fn validate_list_windows_output(value: serde_json::Value) -> Result<(), String> {
- let windows = value
- .get("windows")
- .and_then(serde_json::Value::as_array)
- .ok_or_else(|| "windows must be an array".to_owned())?;
- for (index, window) in windows.iter().enumerate() {
- let z_index = window
- .get("z_index")
- .ok_or_else(|| format!("windows[{index}].z_index is required"))?;
- if !(z_index.is_null() || z_index.is_u64() || z_index.is_i64()) {
- return Err(format!(
- "windows[{index}].z_index must be an integer or null"
- ));
- }
- }
- Ok(())
+fn get_window_state() -> ToolContract {
+ contract::<GetWindowStateInput, WindowStateOutput>(
+ "get_window_state",
+ "Observe an exact native window with snapshot-bound elements and optional screenshots.",
+ &[
+ "accessibility.window_state",
+ "accessibility.tree",
+ "accessibility.tree.structured",
+ "accessibility.tree.bounded",
+ "accessibility.element_tokens",
+ "screen.capture",
+ "screen.capture.window",
+ ],
+ ToolAnnotations {
+ read_only: true,
+ destructive: false,
+ idempotent: false,
+ open_world: false,
+ },
+ CursorAction::Observe,
+ )
}
fn clipboard_read() -> ToolContract {
@@ -235,11 +244,12 @@ fn invoke_menu() -> ToolContract {
fn click() -> ToolContract {
contract::<ClickInput, ActionResult>(
"click",
- "Click an absolute point in get_desktop_state coordinates without targeting a window.",
+ "Click coordinates or a snapshot-bound element with an explicit target and delivery mode.",
&[
"input.pointer.click",
"input.pointer.click.left",
"accessibility.element_tokens",
+ "input.delivery_mode",
],
ToolAnnotations {
read_only: false,
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
index b095bcfaee..69bd68f715 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
@@ -14,6 +14,10 @@ use serde_json::Value;
pub trait ToolInput: Serialize + DeserializeOwned + JsonSchema {
const TOOL_NAME: &'static str;
+ fn validate(&self) -> Result<(), String> {
+ Ok(())
+ }
+
fn input_schema() -> Value {
let settings = schemars::generate::SchemaSettings::draft2020_12().with(|settings| {
settings.meta_schema = None;
@@ -560,9 +564,9 @@ impl ToolInput for MoveCursorInput {
const TOOL_NAME: &'static str = "move_cursor";
}
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(deny_unknown_fields)]
-pub struct ClickInput {
+pub struct LegacyClickInput {
#[schemars(schema_with = "number_schema")]
pub x: f64,
#[schemars(schema_with = "number_schema")]
@@ -586,8 +590,139 @@ pub struct ClickInput {
pub count: Option<u32>,
}
+impl ToolInput for LegacyClickInput {
+ const TOOL_NAME: &'static str = "click";
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)]
+#[serde(rename_all = "snake_case")]
+pub enum InputDeliveryMode {
+ Background,
+ Foreground,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Enum)]
+#[serde(untagged)]
+pub enum ClickPosition {
+ Coordinates { x: f64, y: f64 },
+ Element { element_token: String },
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, uniffi::Record)]
+#[serde(try_from = "ClickWireInput")]
+pub struct ClickInput {
+ pub target: ActionTarget,
+ #[serde(flatten)]
+ pub position: ClickPosition,
+ pub delivery_mode: InputDeliveryMode,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub session: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub button: Option<ClickButton>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub count: Option<u32>,
+}
+
+// Parse the flat wire shape before constructing the sum type: an untagged
+// serde enum alone would silently accept mixed coordinate and element fields.
+#[derive(Debug, Deserialize, JsonSchema)]
+#[serde(deny_unknown_fields)]
+struct ClickWireInput {
+ target: ActionTarget,
+ delivery_mode: InputDeliveryMode,
+ #[serde(default, deserialize_with = "present_click_field")]
+ #[schemars(schema_with = "number_schema")]
+ x: Option<f64>,
+ #[serde(default, deserialize_with = "present_click_field")]
+ #[schemars(schema_with = "number_schema")]
+ y: Option<f64>,
+ #[serde(default, deserialize_with = "present_click_field")]
+ #[schemars(schema_with = "string_schema")]
+ element_token: Option<String>,
+ /// For multi-call work, prefer a short public session label and repeat it on every call that
+ /// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
+ #[serde(default)]
+ #[schemars(schema_with = "string_schema")]
+ session: Option<String>,
+ #[serde(default)]
+ #[schemars(schema_with = "click_button_schema")]
+ button: Option<ClickButton>,
+ #[serde(default)]
+ #[schemars(schema_with = "click_count_schema")]
+ count: Option<u32>,
+}
+
+fn present_click_field<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
+where
+ D: serde::Deserializer<'de>,
+ T: Deserialize<'de>,
+{
+ T::deserialize(deserializer).map(Some)
+}
+
+impl TryFrom<ClickWireInput> for ClickInput {
+ type Error = String;
+ fn try_from(wire: ClickWireInput) -> Result<Self, Self::Error> {
+ let position = match (wire.x, wire.y, wire.element_token) {
+ (Some(x), Some(y), None) => ClickPosition::Coordinates { x, y },
+ (None, None, Some(element_token)) => ClickPosition::Element { element_token },
+ _ => return Err("click requires exactly x and y, or element_token".into()),
+ };
+ let input = Self {
+ target: wire.target,
+ position,
+ delivery_mode: wire.delivery_mode,
+ session: wire.session,
+ button: wire.button,
+ count: wire.count,
+ };
+ input.validate()?;
+ Ok(input)
+ }
+}
+
+impl JsonSchema for ClickInput {
+ fn schema_name() -> std::borrow::Cow<'static, str> {
+ "ClickInput".into()
+ }
+ fn json_schema(generator: &mut SchemaGenerator) -> Schema {
+ let mut schema = ClickWireInput::json_schema(generator);
+ schema.insert("oneOf".into(), serde_json::json!([
+ {"required":["x","y"], "not":{"required":["element_token"]}},
+ {"required":["element_token"], "not":{"anyOf":[{"required":["x"]},{"required":["y"]}]}}
+ ]));
+ schema
+ }
+}
+
impl ToolInput for ClickInput {
const TOOL_NAME: &'static str = "click";
+ fn validate(&self) -> Result<(), String> {
+ match &self.position {
+ ClickPosition::Coordinates { x, y } if !x.is_finite() || !y.is_finite() => {
+ return Err("click coordinates must be finite".into())
+ }
+ ClickPosition::Element { element_token } if element_token.trim().is_empty() => {
+ return Err("element_token must not be empty".into())
+ }
+ _ => {}
+ }
+ if let ActionTarget::Desktop { display_id } = &self.target {
+ if display_id != "primary" {
+ return Err("portable desktop target must be primary".into());
+ }
+ if self.delivery_mode != InputDeliveryMode::Foreground {
+ return Err("desktop clicks require foreground delivery".into());
+ }
+ if matches!(self.position, ClickPosition::Element { .. }) {
+ return Err("element clicks require an exact window target".into());
+ }
+ }
+ if self.count.is_some_and(|count| !(1..=3).contains(&count)) {
+ return Err("click count must be between 1 and 3".into());
+ }
+ Ok(())
+ }
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
@@ -779,8 +914,71 @@ mod tests {
use super::*;
#[test]
- fn generated_click_schema_matches_driver_dialect() {
+ fn typed_click_round_trips_flat_native_wire_and_exact_window_id() {
+ for position in [json!({"x":-1.5,"y":2.0}), json!({"element_token":"s1:0"})] {
+ let mut wire = json!({"target":{"kind":"window","pid":7,"window_id":9007199254740993_u64},"delivery_mode":"background"});
+ wire.as_object_mut()
+ .unwrap()
+ .extend(position.as_object().unwrap().clone());
+ let input: ClickInput = serde_json::from_value(wire.clone()).unwrap();
+ assert_eq!(serde_json::to_value(input).unwrap(), wire);
+ }
let schema = ClickInput::input_schema();
+ assert_eq!(schema["required"], json!(["target", "delivery_mode"]));
+ assert!(schema["oneOf"].is_array());
+ assert!(schema["properties"].get("position").is_none());
+ }
+
+ #[test]
+ fn typed_click_rejects_ambiguous_missing_or_invalid_positions() {
+ for position in [
+ json!({}),
+ json!({"x":1}),
+ json!({"y":2}),
+ json!({"x":1,"y":2,"element_token":"s1:0"}),
+ json!({"x":1,"element_token":"s1:0"}),
+ json!({"x":null,"element_token":"s1:0"}),
+ json!({"element_token":" "}),
+ json!({"x":1,"y":2,"unknown":true}),
+ ] {
+ let mut wire = json!({"target":{"kind":"window","pid":7,"window_id":9},"delivery_mode":"background"});
+ wire.as_object_mut()
+ .unwrap()
+ .extend(position.as_object().unwrap().clone());
+ assert!(
+ serde_json::from_value::<ClickInput>(wire.clone()).is_err(),
+ "{wire}"
+ );
+ }
+ assert!(serde_json::from_value::<ClickInput>(json!({"x":1,"y":2})).is_err());
+ let mut input = ClickInput {
+ target: ActionTarget::Desktop {
+ display_id: "primary".into(),
+ },
+ position: ClickPosition::Coordinates { x: 1.0, y: 2.0 },
+ delivery_mode: InputDeliveryMode::Foreground,
+ session: None,
+ button: None,
+ count: None,
+ };
+ assert!(input.validate().is_ok());
+ input.position = ClickPosition::Coordinates {
+ x: f64::NAN,
+ y: 2.0,
+ };
+ assert!(input.validate().is_err());
+ input.position = ClickPosition::Element {
+ element_token: "s1:0".into(),
+ };
+ assert!(input.validate().is_err());
+ input.position = ClickPosition::Coordinates { x: 1.0, y: 2.0 };
+ input.delivery_mode = InputDeliveryMode::Background;
+ assert!(input.validate().is_err());
+ }
+
+ #[test]
+ fn legacy_click_schema_matches_coordinate_driver_dialect() {
+ let schema = LegacyClickInput::input_schema();
assert_eq!(schema["type"], "object");
assert_eq!(schema["additionalProperties"], false);
assert_eq!(schema["required"], json!(["x", "y"]));
@@ -812,7 +1010,7 @@ mod tests {
#[test]
fn serde_and_schema_reject_unknown_fields() {
- let error = serde_json::from_value::<ClickInput>(json!({
+ let error = serde_json::from_value::<LegacyClickInput>(json!({
"x": 1,
"y": 2,
"pid": 3
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
index b568c0bb12..af64059418 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
@@ -21,20 +21,22 @@ mod inputs;
mod outputs;
mod session;
mod verification;
+mod windows;
+pub use windows::*;
pub use cursor::{
classify_cursor_semantics, CursorAction, CursorDelivery, CursorPlayback, CursorReducedMotion,
CursorSemantics, CursorTarget, CursorThemeSelection,
};
pub use inputs::{
- action_target_schema, ActionTarget, CaptureScope, ClickButton, ClickInput, ClipboardReadInput,
- ClipboardWriteInput, DesktopScope, DragInput, EndSessionInput, EscalateSessionInput,
- EscalationReason, GetAgentCursorStateInput, GetCursorPositionInput, GetDesktopStateInput,
- GetScreenSizeInput, GetSessionInput, GetSessionStateInput, HotkeyInput, InvokeMenuInput,
- ListSessionsInput, MoveCursorInput, PressKeyInput, ScrollBy, ScrollDirection, ScrollInput,
- SetAgentCursorEnabledInput, SetAgentCursorMotionInput, SetAgentCursorThemeInput,
- SetWindowFrameInput, StartSessionInput, ToolInput, TypeTextInput,
- MULTI_CALL_SESSION_DESCRIPTION,
+ action_target_schema, ActionTarget, CaptureScope, ClickButton, ClickInput, ClickPosition,
+ ClipboardReadInput, ClipboardWriteInput, DesktopScope, DragInput, EndSessionInput,
+ EscalateSessionInput, EscalationReason, GetAgentCursorStateInput, GetCursorPositionInput,
+ GetDesktopStateInput, GetScreenSizeInput, GetSessionInput, GetSessionStateInput, HotkeyInput,
+ InputDeliveryMode, InvokeMenuInput, LegacyClickInput, ListSessionsInput, MoveCursorInput,
+ PressKeyInput, ScrollBy, ScrollDirection, ScrollInput, SetAgentCursorEnabledInput,
+ SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetWindowFrameInput, StartSessionInput,
+ ToolInput, TypeTextInput, MULTI_CALL_SESSION_DESCRIPTION,
};
pub use outputs::{
advertised_output_schema, refusal_envelope_schema, ActionDelivery, ActionDeliveryMode,
@@ -60,7 +62,7 @@ pub const TOOLS_LIST_SCHEMA_VERSION: &str = "1";
pub const CAPABILITY_VERSION: &str = "1";
/// Shape version for the checked-in generated client contract.
-pub const CONTRACT_VERSION: &str = "0.7.0";
+pub const CONTRACT_VERSION: &str = "0.8.0";
/// MCP protocol version used by current cua-driver clients.
pub const MCP_PROTOCOL_VERSION: &str = "2025-06-18";
@@ -257,9 +259,6 @@ pub fn tool_input_fields(name: &str) -> Option<&'static BTreeSet<String>> {
/// MCP tool. Runtime-only tools can define a narrow shared schema here without
/// committing every generated SDK to their broader platform-specific shape.
pub fn tool_success_output_schema(name: &str) -> Option<Value> {
- if name == "list_windows" {
- return Some(desktop::list_windows_success_output_schema());
- }
tool_contract(name).and_then(|contract| contract.success_output_schema)
}
@@ -270,10 +269,6 @@ pub fn validate_success_output(name: &str, value: Value) -> Result<bool, String>
validate_typed_output::<ActionResult>(value)?;
return Ok(true);
}
- if name == "list_windows" {
- desktop::validate_list_windows_output(value)?;
- return Ok(true);
- }
if let Some(entry) = tool_index().get(name) {
(entry.output_validator)(value)?;
Ok(true)
@@ -297,7 +292,7 @@ mod tests {
let mut sorted = names.clone();
sorted.sort_unstable();
assert_eq!(names, sorted);
- assert_eq!(manifest.contract_version, "0.7.0");
+ assert_eq!(manifest.contract_version, "0.8.0");
assert!(manifest.experimental);
}
@@ -473,7 +468,7 @@ mod tests {
#[test]
fn list_windows_defines_nullable_higher_is_frontmost_z_index() {
- assert!(tool_contract("list_windows").is_none());
+ assert!(tool_contract("list_windows").is_some());
let schema = tool_success_output_schema("list_windows").expect("runtime schema");
let z_index = &schema["properties"]["windows"]["items"]["properties"]["z_index"];
assert_eq!(z_index["type"], serde_json::json!(["integer", "null"]));
@@ -486,8 +481,7 @@ mod tests {
"list_windows",
serde_json::json!({
"windows": [
- {"z_index": 4, "platform_field": true},
- {"z_index": null}
+ {"window_id":1,"pid":2,"app_name":"Example","title":"Doc","bounds":{"x":0,"y":0,"width":10,"height":10},"is_on_screen":true,"z_index":null,"platform_field":true}
],
"current_space_id": null
}),
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
index 73870e1283..5caea810f8 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
@@ -55,7 +55,9 @@ pub fn advertised_output_schema(success: Value) -> Value {
serde_json::json!({ "type": "object", "anyOf": [success, refusal_envelope_schema()] })
}
-fn output_schema_with_additional_properties<T: JsonSchema>(additional_properties: bool) -> Value {
+pub(crate) fn output_schema_with_additional_properties<T: JsonSchema>(
+ additional_properties: bool,
+) -> Value {
let mut settings = SchemaSettings::draft2020_12();
settings.inline_subschemas = true;
settings.meta_schema = None;
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/windows.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/windows.rs
new file mode 100644
index 0000000000..49dea44b0f
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/windows.rs
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Portable native application discovery and exact-window observations.
+
+use crate::{ToolInput, ToolOutput};
+use schemars::{json_schema, JsonSchema, Schema, SchemaGenerator};
+use serde::{Deserialize, Serialize};
+
+fn string_schema(g: &mut SchemaGenerator) -> Schema {
+ String::json_schema(g)
+}
+fn bool_schema(g: &mut SchemaGenerator) -> Schema {
+ bool::json_schema(g)
+}
+fn pid_schema(_: &mut SchemaGenerator) -> Schema {
+ json_schema!({"type":"integer", "minimum":1, "maximum":4294967295_u64})
+}
+fn positive_integer_schema(_: &mut SchemaGenerator) -> Schema {
+ json_schema!({"type":"integer", "minimum":1})
+}
+
+fn nullable_pid_schema(_: &mut SchemaGenerator) -> Schema {
+ json_schema!({"type":["integer","null"], "minimum":0, "maximum":4294967295_u64})
+}
+
+fn nullable_z_index_schema(_: &mut SchemaGenerator) -> Schema {
+ json_schema!({"type":["integer","null"]})
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+#[serde(deny_unknown_fields)]
+pub struct ListAppsInput {}
+
+impl ToolInput for ListAppsInput {
+ const TOOL_NAME: &'static str = "list_apps";
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+#[serde(deny_unknown_fields)]
+pub struct ListWindowsInput {
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "pid_schema")]
+ pub pid: Option<u32>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "bool_schema")]
+ pub on_screen_only: Option<bool>,
+}
+
+impl ToolInput for ListWindowsInput {
+ const TOOL_NAME: &'static str = "list_windows";
+
+ fn validate(&self) -> Result<(), String> {
+ if self.pid == Some(0) {
+ return Err("window discovery requires a positive process ID".into());
+ }
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+#[serde(deny_unknown_fields)]
+pub struct GetWindowStateInput {
+ #[schemars(schema_with = "pid_schema")]
+ pub pid: u32,
+ #[schemars(schema_with = "positive_integer_schema")]
+ pub window_id: u64,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "string_schema")]
+ pub session: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "string_schema")]
+ pub query: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "bool_schema")]
+ pub include_accessibility_tree: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "bool_schema")]
+ pub include_screenshot: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "string_schema")]
+ pub screenshot_out_file: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "positive_integer_schema")]
+ pub max_elements: Option<u32>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "positive_integer_schema")]
+ pub max_depth: Option<u32>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[schemars(schema_with = "positive_integer_schema")]
+ pub max_dimension: Option<u32>,
+}
+
+impl ToolInput for GetWindowStateInput {
+ const TOOL_NAME: &'static str = "get_window_state";
+ fn validate(&self) -> Result<(), String> {
+ if self.pid == 0 || self.window_id == 0 {
+ return Err("window observation requires positive process and window IDs".into());
+ }
+ if [self.max_elements, self.max_depth, self.max_dimension].contains(&Some(0)) {
+ return Err("window observation limits must be positive".into());
+ }
+ if self.include_accessibility_tree == Some(false) && self.include_screenshot == Some(false)
+ {
+ return Err("window observation requires accessibility or screenshot capture".into());
+ }
+ Ok(())
+ }
+}
+
+fn required_nullable<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
+where
+ D: serde::Deserializer<'de>,
+ T: Deserialize<'de>,
+{
+ Option::<T>::deserialize(deserializer)
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct AppInfo {
+ pub pid: u32,
+ pub name: String,
+ pub running: bool,
+ pub active: bool,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub bundle_id: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub launch_path: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub kind: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub last_used: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct ListAppsOutput {
+ pub apps: Vec<AppInfo>,
+}
+
+impl ToolOutput for ListAppsOutput {}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct WindowBounds {
+ pub x: f64,
+ pub y: f64,
+ pub width: f64,
+ pub height: f64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct WindowInfo {
+ pub window_id: u64,
+ #[serde(deserialize_with = "required_nullable")]
+ #[schemars(required, schema_with = "nullable_pid_schema")]
+ pub pid: Option<u32>,
+ pub app_name: String,
+ pub title: String,
+ pub bounds: WindowBounds,
+ pub is_on_screen: bool,
+ #[serde(deserialize_with = "required_nullable")]
+ #[schemars(required, schema_with = "nullable_z_index_schema")]
+ /// Higher values are closer to the front. Null means stacking order is unavailable; callers must not infer an order from array position.
+ pub z_index: Option<i64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub layer: Option<i32>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub minimized: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub current_space_id: Option<u64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub on_current_space: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub space_ids: Option<Vec<u64>>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct ListWindowsOutput {
+ pub windows: Vec<WindowInfo>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub current_space_id: Option<u64>,
+}
+
+impl ToolOutput for ListWindowsOutput {
+ fn output_schema() -> serde_json::Value {
+ let mut schema = crate::outputs::output_schema_with_additional_properties::<Self>(true);
+ // Stacking semantics are part of the existing live discovery contract.
+ schema["properties"]["windows"]["items"]["properties"]["z_index"]["description"] =
+ serde_json::json!("Higher values are closer to the front. Null means the provider cannot observe stacking order; callers must not infer an order from array position or treat null as zero.");
+ schema
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct ElementFrame {
+ pub x: f64,
+ pub y: f64,
+ pub w: f64,
+ pub h: f64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct WindowElement {
+ pub element_index: u64,
+ pub role: String,
+ pub depth: u32,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub element_token: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub label: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub value: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub value_description: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub enabled: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub selected: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub in_web_content: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub actions: Option<Vec<String>>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub parent_index: Option<u64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub frame: Option<ElementFrame>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub min: Option<f64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub max: Option<f64>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct SnapshotImage {
+ pub mime_type: String,
+ pub data_base64: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
+pub struct WindowStateOutput {
+ pub pid: u32,
+ pub window_id: u64,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub snapshot_id: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub app_name: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub window_title: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub tree_markdown: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub elements: Option<Vec<WindowElement>>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub element_count: Option<u64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub total_element_count: Option<u64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub returned_element_count: Option<u64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub filtered_element_count: Option<u64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub elements_complete: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub degraded: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub degraded_reason: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub truncated: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub truncation_reason: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub screenshot_width: Option<u32>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub screenshot_height: Option<u32>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub screenshot_scale: Option<f64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub screenshot_mime_type: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub screenshot_file_path: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub screenshot_frame_valid: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub window_bounds: Option<WindowBounds>,
+ /// Image content belongs to the MCP envelope, never structuredContent.
+ #[serde(skip)]
+ #[schemars(skip)]
+ pub images: Vec<SnapshotImage>,
+}
+
+impl ToolOutput for WindowStateOutput {
+ fn validate(&self) -> Result<(), String> {
+ match (self.screenshot_width, self.screenshot_height) {
+ (None, None) => {}
+ (Some(width), Some(height)) if width > 0 && height > 0 => {}
+ _ => return Err("screenshot dimensions must be a positive width/height pair".into()),
+ }
+ if self
+ .screenshot_scale
+ .is_some_and(|scale| !scale.is_finite() || scale <= 0.0)
+ {
+ return Err("screenshot_scale must be finite and positive".into());
+ }
+ if let (Some(elements), Some(returned)) = (&self.elements, self.returned_element_count) {
+ if elements.len() as u64 != returned {
+ return Err("returned_element_count does not match elements".into());
+ }
+ }
+ if let (Some(total), Some(returned)) =
+ (self.total_element_count, self.returned_element_count)
+ {
+ if returned > total {
+ return Err("returned_element_count exceeds total_element_count".into());
+ }
+ }
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn native_app_success_shapes_allow_unavailable_metadata() {
+ for metadata in [
+ json!({"bundle_id":"org.example.App", "kind":"desktop", "launch_path":"/Applications/Example.app", "last_used":null}),
+ json!({"bundle_id":null, "kind":"desktop", "launch_path":"C:\\Example.exe", "last_used":null}),
+ json!({"bundle_id":null, "kind":null, "launch_path":null, "last_used":null}),
+ ] {
+ let mut app =
+ json!({"pid":7,"name":"Example","running":true,"active":false,"windows":[]});
+ app.as_object_mut()
+ .unwrap()
+ .extend(metadata.as_object().unwrap().clone());
+ let output: ListAppsOutput = serde_json::from_value(json!({"apps":[app]})).unwrap();
+ assert_eq!(output.apps[0].pid, 7);
+ }
+ }
+
+ #[test]
+ fn native_window_success_shapes_preserve_unknown_pid_and_order() {
+ for metadata in [
+ json!({"pid":7,"z_index":4,"layer":0,"space_ids":[1],"current_space_id":1,"on_current_space":true}),
+ json!({"pid":7,"z_index":0,"layer":0,"minimized":false}),
+ json!({"pid":null,"z_index":null,"x":-10,"y":0,"width":100,"height":80}),
+ ] {
+ let mut window = json!({"window_id":9007199254740993_u64,"app_name":"Example","title":"Document","bounds":{"x":-10,"y":0,"width":100,"height":80},"is_on_screen":true});
+ window
+ .as_object_mut()
+ .unwrap()
+ .extend(metadata.as_object().unwrap().clone());
+ let output: ListWindowsOutput =
+ serde_json::from_value(json!({"windows":[window]})).unwrap();
+ assert_eq!(output.windows[0].window_id, 9007199254740993);
+ }
+ let missing = json!({"windows":[{"window_id":1,"pid":null,"app_name":"Example","title":"Document","bounds":{"x":0,"y":0,"width":100,"height":80},"is_on_screen":true}]});
+ assert!(serde_json::from_value::<ListWindowsOutput>(missing).is_err());
+ }
+
+ #[test]
+ fn window_state_supports_capture_only_and_degraded_native_shapes() {
+ let capture: WindowStateOutput = serde_json::from_value(json!({
+ "pid":7,"window_id":9,"screenshot_width":100,"screenshot_height":80,
+ "screenshot_mime_type":"image/png","screenshot_file_path":"/tmp/example.png",
+ "window_bounds":{"x":0,"y":0,"width":100,"height":80},"screenshot_scale":1.0
+ }))
+ .unwrap();
+ assert!(capture.elements.is_none());
+ assert!(capture.images.is_empty());
+ for value in [
+ json!({"pid":7,"window_id":9,"element_count":0,"elements":[],"tree_markdown":"","elements_complete":false,"degraded":true,"degraded_reason":"ax_window_unresolved"}),
+ json!({"pid":7,"window_id":9,"element_count":1,"elements":[{"element_index":0,"role":"button","depth":1,"element_token":"s1:0","label":"Apply","frame":{"x":10,"y":20,"w":30,"h":40},"enabled":true}],"elements_complete":false,"screenshot_error":"capture unavailable"}),
+ json!({"pid":7,"window_id":9,"elements":[{"element_index":0,"role":"button","depth":1}],"degraded":true,"degraded_reason":"accessibility_window_identity_unproven"}),
+ ] {
+ let output: WindowStateOutput = serde_json::from_value(value).unwrap();
+ assert!(output.elements.is_some());
+ output.validate().unwrap();
+ }
+ }
+
+ #[test]
+ fn images_are_envelope_only_and_not_structured_schema_fields() {
+ let mut output: WindowStateOutput =
+ serde_json::from_value(json!({"pid":7,"window_id":9})).unwrap();
+ output.images.push(SnapshotImage {
+ mime_type: "image/png".into(),
+ data_base64: "aGVsbG8=".into(),
+ });
+ assert!(serde_json::to_value(&output)
+ .unwrap()
+ .get("images")
+ .is_none());
+ assert!(WindowStateOutput::output_schema()["properties"]
+ .get("images")
+ .is_none());
+ assert_eq!(
+ GetWindowStateInput::input_schema()["properties"]["max_elements"]["minimum"],
+ 1
+ );
+ assert_eq!(ListAppsInput::input_schema()["properties"], json!({}));
+ }
+
+ #[test]
+ fn invalid_window_observation_metadata_fails_validation() {
+ for extra in [
+ json!({"screenshot_width":100}),
+ json!({"screenshot_width":0,"screenshot_height":80}),
+ json!({"screenshot_scale":0}),
+ json!({"elements":[],"returned_element_count":1}),
+ json!({"total_element_count":1,"returned_element_count":2}),
+ ] {
+ let mut value = json!({"pid":7,"window_id":9});
+ value
+ .as_object_mut()
+ .unwrap()
+ .extend(extra.as_object().unwrap().clone());
+ let output: WindowStateOutput = serde_json::from_value(value).unwrap();
+ assert!(output.validate().is_err());
+ }
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index 0cb5548da1..b3ea0b6e4d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -2937,10 +2937,13 @@ mod runtime_isolation_tests {
&self.def
}
- async fn invoke(&self, _args: serde_json::Value) -> crate::protocol::ToolResult {
+ async fn invoke(&self, args: serde_json::Value) -> crate::protocol::ToolResult {
self.hits.fetch_add(1, Ordering::SeqCst);
- crate::protocol::ToolResult::text("private state")
- .with_structured(serde_json::json!({"snapshot_id": 1}))
+ crate::protocol::ToolResult::text("private state").with_structured(serde_json::json!({
+ "pid": args["pid"].as_u64().unwrap_or(42),
+ "window_id": args["window_id"].as_u64().unwrap_or(7),
+ "snapshot_id": "synthetic-snapshot", "elements": []
+ }))
}
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool_args.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool_args.rs
index 883e9e9306..c045a8ebaa 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool_args.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool_args.rs
@@ -78,6 +78,23 @@ pub fn parse_typed_projection<T: cua_driver_contract::ToolInput>(
parse_typed_input(tool_name, Value::Object(projected))
}
+/// Keep the released desktop-coordinate wire parser independent of the SDK's
+/// explicit addressing API. Native handlers still accept legacy scope/defaults.
+pub fn parse_legacy_click_input(
+ args: &Value,
+) -> Result<cua_driver_contract::LegacyClickInput, ToolResult> {
+ const FIELDS: &[&str] = &["x", "y", "target", "scope", "session", "button", "count"];
+ let Some(source) = args.as_object() else {
+ return parse_typed_input("click", args.clone());
+ };
+ let projected = source
+ .iter()
+ .filter(|(name, _)| FIELDS.contains(&name.as_str()))
+ .map(|(name, value)| (name.clone(), value.clone()))
+ .collect();
+ parse_typed_input("click", Value::Object(projected))
+}
+
/// Remove transport-reserved arguments supplied by a public caller. Trusted
/// ingress code calls this before injecting session or approval context, so a
/// tool request cannot forge underscore-prefixed transport evidence.
@@ -89,8 +106,8 @@ pub fn sanitize_reserved_args(args: &mut Value) {
#[cfg(test)]
mod reserved_args_tests {
- use super::{parse_typed_input, parse_typed_projection, sanitize_reserved_args};
- use cua_driver_contract::{ClickButton, ClickInput, GetScreenSizeInput, SetWindowFrameInput};
+ use super::{parse_legacy_click_input, parse_typed_input, sanitize_reserved_args};
+ use cua_driver_contract::{ClickButton, GetScreenSizeInput, SetWindowFrameInput};
use serde_json::json;
#[test]
@@ -159,27 +176,23 @@ mod reserved_args_tests {
}
#[test]
- fn typed_projection_preserves_rich_platform_fields_without_bypassing_portable_types() {
- let input = parse_typed_projection::<ClickInput>(
- "click",
- &json!({
- "x": 10,
- "y": 20.5,
- "scope": "desktop",
- "button": "right",
- "pid": 42,
- "delivery_mode": "foreground"
- }),
- )
+ fn legacy_click_projection_preserves_wire_defaults_and_validates_coordinates() {
+ let input = parse_legacy_click_input(&json!({
+ "x": 10,
+ "y": 20.5,
+ "scope": "desktop",
+ "button": "right",
+ "pid": 42,
+ "delivery_mode": "foreground"
+ }))
.expect("rich fields are projected away");
assert_eq!(input.x, 10.0);
assert_eq!(input.button, Some(ClickButton::Right));
- let error = parse_typed_projection::<ClickInput>(
- "click",
- &json!({"x": "10", "y": 20, "scope": "desktop", "pid": 42}),
- )
- .expect_err("portable field types are still enforced");
+ assert!(input.target.is_none());
+ let error =
+ parse_legacy_click_input(&json!({"x": "10", "y": 20, "scope": "desktop", "pid": 42}))
+ .expect_err("portable field types are still enforced");
assert_eq!(error.is_error, Some(true));
assert_eq!(
error
diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
index af0e9d5d22..87d18f66fd 100644
--- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
@@ -8,11 +8,13 @@
use cua_driver_contract::{
ActionResult, ClickInput, ClipboardReadInput, ClipboardWriteInput, DragInput, EndSessionInput,
EndSessionOutput, EscalateSessionInput, GetAgentCursorStateInput, GetCursorPositionInput,
- GetDesktopStateInput, GetScreenSizeInput, GetSessionInput, GetSessionStateInput, HotkeyInput,
- InvokeMenuInput, ListSessionsInput, ListSessionsOutput, MoveCursorInput, PressKeyInput,
- ScrollInput, SessionOutput, SessionStateOutput, SetAgentCursorEnabledInput,
- SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetWindowFrameInput, StartSessionInput,
- StartSessionOutput, ToolInput, TypeTextInput, VerifyStateInput, VerifyStateOutput,
+ GetDesktopStateInput, GetScreenSizeInput, GetSessionInput, GetSessionStateInput,
+ GetWindowStateInput, HotkeyInput, InvokeMenuInput, ListAppsInput, ListAppsOutput,
+ ListSessionsInput, ListSessionsOutput, ListWindowsInput, ListWindowsOutput, MoveCursorInput,
+ PressKeyInput, ScrollInput, SessionOutput, SessionStateOutput, SetAgentCursorEnabledInput,
+ SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetWindowFrameInput, SnapshotImage,
+ StartSessionInput, StartSessionOutput, ToolInput, ToolOutput, TypeTextInput, VerifyStateInput,
+ VerifyStateOutput, WindowStateOutput,
};
use cua_driver_core::daemon::{
is_daemon_listening, request_daemon_metadata, send_request, socket_path_for_namespace,
@@ -637,7 +639,6 @@ macro_rules! desktop_tool_methods {
move_cursor: MoveCursorInput,
set_window_frame: SetWindowFrameInput,
invoke_menu: InvokeMenuInput,
- click: ClickInput,
drag: DragInput,
scroll: ScrollInput,
clipboard_read: ClipboardReadInput,
@@ -689,6 +690,10 @@ macro_rules! define_exported_tool_names {
<ListSessionsInput as ToolInput>::TOOL_NAME,
<GetSessionStateInput as ToolInput>::TOOL_NAME,
<EndSessionInput as ToolInput>::TOOL_NAME,
+ <ListAppsInput as ToolInput>::TOOL_NAME,
+ <ListWindowsInput as ToolInput>::TOOL_NAME,
+ <GetWindowStateInput as ToolInput>::TOOL_NAME,
+ <ClickInput as ToolInput>::TOOL_NAME,
$(<$input as ToolInput>::TOOL_NAME,)*
];
};
@@ -696,6 +701,49 @@ macro_rules! define_exported_tool_names {
desktop_tool_methods!(define_exported_tool_names);
+macro_rules! define_native_window_methods {
+ ($driver:ty) => {
+ #[uniffi::export(async_runtime = "tokio")]
+ impl $driver {
+ pub async fn list_apps(
+ &self,
+ input: ListAppsInput,
+ ) -> Result<ListAppsOutput, DriverError> {
+ self.invoke_typed(ListAppsInput::TOOL_NAME, input)
+ .await?
+ .typed_success(ListAppsInput::TOOL_NAME)
+ }
+
+ pub async fn list_windows(
+ &self,
+ input: ListWindowsInput,
+ ) -> Result<ListWindowsOutput, DriverError> {
+ self.invoke_typed(ListWindowsInput::TOOL_NAME, input)
+ .await?
+ .typed_success(ListWindowsInput::TOOL_NAME)
+ }
+
+ pub async fn get_window_state(
+ &self,
+ input: GetWindowStateInput,
+ ) -> Result<WindowStateOutput, DriverError> {
+ self.invoke_typed(GetWindowStateInput::TOOL_NAME, input)
+ .await?
+ .window_state_success()
+ }
+
+ pub async fn click(&self, input: ClickInput) -> Result<ActionResult, DriverError> {
+ self.invoke_typed(ClickInput::TOOL_NAME, input)
+ .await?
+ .typed_success(ClickInput::TOOL_NAME)
+ }
+ }
+ };
+}
+
+define_native_window_methods!(CuaDriver);
+define_native_window_methods!(CuaDriverSession);
+
#[uniffi::export]
impl CuaDriver {
#[doc(hidden)]
@@ -1542,11 +1590,17 @@ impl CuaDriverSession {
}
impl CuaDriverSession {
- async fn invoke_typed<T: Serialize>(
+ async fn invoke_typed<T: ToolInput>(
&self,
name: &str,
input: T,
) -> Result<ToolResult, DriverError> {
+ input
+ .validate()
+ .map_err(|reason| DriverError::InvalidArguments {
+ tool: name.into(),
+ reason,
+ })?;
let arguments =
serde_json::to_value(input).map_err(|error| DriverError::InvalidArguments {
tool: name.into(),
@@ -1619,11 +1673,17 @@ impl CuaDriver {
self.invoke(name, arguments).await
}
- async fn invoke_typed<T: Serialize>(
+ async fn invoke_typed<T: ToolInput>(
&self,
name: &str,
input: T,
) -> Result<ToolResult, DriverError> {
+ input
+ .validate()
+ .map_err(|reason| DriverError::InvalidArguments {
+ tool: name.into(),
+ reason,
+ })?;
let arguments =
serde_json::to_value(input).map_err(|error| DriverError::InvalidArguments {
tool: name.into(),
@@ -1698,20 +1758,103 @@ impl ToolResult {
self.verification.as_ref()
}
- fn typed_success<T: serde::de::DeserializeOwned>(self, tool: &str) -> Result<T, DriverError> {
+ fn typed_success<T: ToolOutput>(self, tool: &str) -> Result<T, DriverError> {
if self.is_error {
return Err(DriverError::Tool {
tool: tool.into(),
- message: self.text,
+ message: self.refusal_message(),
error_code: self.error_code.unwrap_or_default(),
});
}
let structured = self.structured_json.ok_or_else(|| DriverError::Protocol {
reason: format!("{tool} response omitted structuredContent"),
})?;
- serde_json::from_str(&structured).map_err(|error| DriverError::Protocol {
- reason: format!("{tool} returned an invalid typed result: {error}"),
- })
+ let output: T =
+ serde_json::from_str(&structured).map_err(|error| DriverError::Protocol {
+ reason: format!("{tool} returned an invalid typed result: {error}"),
+ })?;
+ output.validate().map_err(|reason| DriverError::Protocol {
+ reason: format!("{tool} returned an invalid typed result: {reason}"),
+ })?;
+ Ok(output)
+ }
+
+ fn refusal_message(&self) -> String {
+ self.structured_json
+ .as_deref()
+ .and_then(|json| serde_json::from_str::<Value>(json).ok())
+ .and_then(|value| {
+ value
+ .get("message")
+ .and_then(Value::as_str)
+ .or_else(|| value.get("refusal")?.get("message")?.as_str())
+ .map(str::to_owned)
+ })
+ .unwrap_or_else(|| self.text.clone())
+ }
+
+ fn window_state_success(mut self) -> Result<WindowStateOutput, DriverError> {
+ if !self.is_error {
+ // Normalization preserves open-ended results; this typed boundary must
+ // not silently discard a malformed image part from a snapshot.
+ let raw: Value =
+ serde_json::from_str(&self.raw_json).map_err(|error| DriverError::Protocol {
+ reason: format!("get_window_state returned an invalid envelope: {error}"),
+ })?;
+ if let Some(content) = raw.get("content") {
+ let content = content.as_array().ok_or_else(|| DriverError::Protocol {
+ reason: "get_window_state content must be an array".into(),
+ })?;
+ for part in content {
+ if part.get("type").and_then(Value::as_str) == Some("image") {
+ let valid = part
+ .get("mimeType")
+ .and_then(Value::as_str)
+ .is_some_and(|mime| mime.starts_with("image/") && mime.len() > 6)
+ && part
+ .get("data")
+ .and_then(Value::as_str)
+ .is_some_and(|data| !data.is_empty());
+ if !valid {
+ return Err(DriverError::Protocol {
+ reason: "get_window_state returned a malformed image part".into(),
+ });
+ }
+ }
+ }
+ }
+ if !self.images.is_empty() {
+ let metadata = &raw["structuredContent"];
+ for dimension in ["screenshot_width", "screenshot_height"] {
+ if !metadata[dimension].as_u64().is_some_and(|value| value > 0) {
+ return Err(DriverError::Protocol {
+ reason: format!("get_window_state image omitted valid {dimension}"),
+ });
+ }
+ }
+ if let Some(mime) = metadata["screenshot_mime_type"].as_str() {
+ if self.images.iter().any(|image| image.mime_type != mime) {
+ return Err(DriverError::Protocol {
+ reason: "get_window_state image MIME type disagrees with metadata"
+ .into(),
+ });
+ }
+ }
+ }
+ }
+ let images = std::mem::take(&mut self.images);
+ let mut output: WindowStateOutput = self.typed_success(GetWindowStateInput::TOOL_NAME)?;
+ output.images = images
+ .into_iter()
+ .map(|image| SnapshotImage {
+ mime_type: image.mime_type,
+ data_base64: image.data_base64,
+ })
+ .collect();
+ output.validate().map_err(|reason| DriverError::Protocol {
+ reason: format!("get_window_state returned an invalid typed result: {reason}"),
+ })?;
+ Ok(output)
}
}
@@ -1829,6 +1972,7 @@ uniffi::setup_scaffolding!("cua_driver_sdk");
#[cfg(test)]
mod tests {
+ mod native_windows;
use super::*;
#[cfg(unix)]
use std::io::{BufRead, BufReader, Write};
diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/tests/native_windows.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/tests/native_windows.rs
new file mode 100644
index 0000000000..c7c2eeb045
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/tests/native_windows.rs
@@ -0,0 +1,337 @@
+use super::*;
+use cua_driver_contract::{ActionTarget, ClickPosition, InputDeliveryMode};
+use serde_json::json;
+
+fn success(tool: &str, structured: Value) -> ToolResult {
+ normalize_result(
+ tool,
+ json!({"content": [], "structuredContent": structured, "isError": false}),
+ )
+ .unwrap()
+}
+
+#[tokio::test]
+async fn portable_inputs_match_the_in_process_native_registry() {
+ let driver = CuaDriver::create(None).unwrap();
+ let live: Value = serde_json::from_str(&driver.list_tools_json().await.unwrap()).unwrap();
+ let tools = live["tools"].as_array().expect("native tools list");
+ for contract in cua_driver_contract::manifest().tools {
+ let native = tools
+ .iter()
+ .find(|tool| tool["name"] == contract.name)
+ .unwrap_or_else(|| panic!("missing native tool {}", contract.name));
+ let violations = cua_driver_contract::compatibility::schema_subset_violations(
+ &contract.input_schema,
+ &native["inputSchema"],
+ );
+ assert!(violations.is_empty(), "{}: {violations:?}", contract.name);
+ assert_eq!(
+ native["annotations"],
+ json!({
+ "readOnlyHint": contract.annotations.read_only,
+ "destructiveHint": contract.annotations.destructive,
+ "idempotentHint": contract.annotations.idempotent,
+ "openWorldHint": contract.annotations.open_world,
+ }),
+ "{} annotations differ from the portable contract",
+ contract.name
+ );
+ }
+ driver.shutdown().await.unwrap();
+}
+
+#[test]
+fn typed_discovery_decodes_native_platform_variants() {
+ for platform in ["macos", "windows", "x11", "wayland"] {
+ let mut app = json!({"pid": 42, "name": "Editor", "running": true, "active": false});
+ let mut window = json!({
+ "window_id": 73, "pid": 42, "app_name": "Editor", "title": "Document",
+ "bounds": {"x": -20, "y": 10, "width": 600, "height": 400},
+ "is_on_screen": true, "z_index": 2
+ });
+ match platform {
+ "macos" => {
+ app["bundle_id"] = json!("example.editor");
+ window["space_ids"] = json!([1, 2]);
+ window["on_current_space"] = json!(true);
+ }
+ "windows" => window["minimized"] = json!(false),
+ "wayland" => {
+ window["pid"] = Value::Null;
+ window["z_index"] = Value::Null;
+ }
+ _ => {}
+ }
+ let apps: ListAppsOutput = success("list_apps", json!({"apps": [app]}))
+ .typed_success("list_apps")
+ .unwrap();
+ let windows: ListWindowsOutput = success("list_windows", json!({"windows": [window]}))
+ .typed_success("list_windows")
+ .unwrap();
+ assert_eq!(apps.apps[0].pid, 42, "{platform}");
+ assert_eq!(windows.windows[0].window_id, 73, "{platform}");
+ assert_eq!(windows.windows[0].bounds.x, -20.0);
+ assert_eq!(windows.windows[0].z_index.is_none(), platform == "wayland");
+ }
+}
+
+#[test]
+fn typed_methods_preserve_both_native_refusal_shapes() {
+ for refusal in [
+ json!({"code":"surface_identity_unproven", "message":"Window is not proven", "effect":"refused"}),
+ json!({"status":"refused", "refusal":{"code":"surface_identity_unproven", "message":"Window is not proven"}}),
+ ] {
+ for tool in ["list_apps", "list_windows", "get_window_state", "click"] {
+ let result = normalize_result(
+ tool,
+ json!({
+ "isError":true, "content":[{"type":"text","text":"Diagnostic details"}],
+ "structuredContent":refusal
+ }),
+ )
+ .unwrap();
+ let error = match tool {
+ "list_apps" => result.typed_success::<ListAppsOutput>(tool).unwrap_err(),
+ "list_windows" => result.typed_success::<ListWindowsOutput>(tool).unwrap_err(),
+ "get_window_state" => result.window_state_success().unwrap_err(),
+ _ => result.typed_success::<ActionResult>(tool).unwrap_err(),
+ };
+ assert!(
+ matches!(error, DriverError::Tool { error_code, message, .. }
+ if error_code == "surface_identity_unproven" && message == "Window is not proven")
+ );
+ }
+ }
+}
+
+#[test]
+fn typed_discovery_rejects_missing_and_malformed_success() {
+ for structured in [
+ Value::Null,
+ json!({}),
+ json!({"apps":"wrong", "windows":42}),
+ ] {
+ assert!(matches!(
+ success("list_apps", structured.clone()).typed_success::<ListAppsOutput>("list_apps"),
+ Err(DriverError::Protocol { .. })
+ ));
+ assert!(matches!(
+ success("list_windows", structured.clone())
+ .typed_success::<ListWindowsOutput>("list_windows"),
+ Err(DriverError::Protocol { .. })
+ ));
+ assert!(matches!(
+ success("get_window_state", structured).window_state_success(),
+ Err(DriverError::Protocol { .. })
+ ));
+ }
+}
+
+fn snapshot_envelope() -> Value {
+ json!({
+ "isError": false,
+ "content": [
+ {"type":"image", "mimeType":"image/png", "data":"cG5n"},
+ {"type":"image", "mimeType":"image/png", "data":"bW9yZQ=="}
+ ],
+ "structuredContent": {
+ "pid":42, "window_id":73, "snapshot_id":"s1",
+ "screenshot_width":600, "screenshot_height":400, "screenshot_mime_type":"image/png",
+ "elements":[{"element_index":0,"element_token":"s1:0","role":"button","depth":0,"label":"Save"}],
+ "degraded":true, "degraded_reason":"Partial tree", "truncated":true
+ }
+ })
+}
+
+#[test]
+fn typed_snapshot_preserves_all_images_and_observation_metadata() {
+ let output = normalize_result("get_window_state", snapshot_envelope())
+ .unwrap()
+ .window_state_success()
+ .unwrap();
+ assert_eq!(output.images.len(), 2);
+ assert_eq!(output.images[0].data_base64, "cG5n");
+ assert_eq!(output.images[1].data_base64, "bW9yZQ==");
+ assert_eq!(output.snapshot_id.as_deref(), Some("s1"));
+ assert_eq!(
+ output.elements.unwrap()[0].element_token.as_deref(),
+ Some("s1:0")
+ );
+ assert_eq!(output.degraded, Some(true));
+ assert_eq!(output.truncated, Some(true));
+}
+
+#[test]
+fn typed_snapshot_rejects_dropped_images_and_inconsistent_metadata() {
+ for (pointer, value) in [
+ ("/content/0/data", Value::Null),
+ ("/content/0/data", json!("")),
+ ("/content/0/mimeType", json!("text/plain")),
+ ("/structuredContent/screenshot_width", json!(0)),
+ ("/structuredContent/screenshot_height", Value::Null),
+ (
+ "/structuredContent/screenshot_mime_type",
+ json!("image/jpeg"),
+ ),
+ ] {
+ let mut envelope = snapshot_envelope();
+ *envelope.pointer_mut(pointer).unwrap() = value;
+ assert!(
+ matches!(
+ normalize_result("get_window_state", envelope)
+ .unwrap()
+ .window_state_success(),
+ Err(DriverError::Protocol { .. })
+ ),
+ "{pointer}"
+ );
+ }
+}
+
+#[test]
+fn typed_snapshot_retains_capture_failure_and_file_only_states() {
+ for metadata in [
+ json!({"pid":42,"window_id":73,"screenshot_frame_valid":false,"degraded":true}),
+ json!({"pid":42,"window_id":73,"screenshot_width":600,"screenshot_height":400,"screenshot_file_path":"/tmp/snapshot.png"}),
+ ] {
+ let output = success("get_window_state", metadata)
+ .window_state_success()
+ .unwrap();
+ assert!(output.images.is_empty());
+ assert!(
+ output.screenshot_frame_valid == Some(false) || output.screenshot_file_path.is_some()
+ );
+ }
+}
+
+#[tokio::test]
+#[cfg(unix)]
+async fn typed_discovery_requests_use_the_existing_daemon_transport() {
+ let (_directory, socket, server) = serve_once(
+ json!({"ok":true,"result":{"structuredContent":{"apps":[]},"content":[],"isError":false}}),
+ );
+ let driver = CuaDriver::connect(Some(socket)).unwrap();
+ assert!(driver
+ .list_apps(ListAppsInput {})
+ .await
+ .unwrap()
+ .apps
+ .is_empty());
+ let request = server.join().unwrap();
+ assert_eq!(request["name"], "list_apps");
+ assert_eq!(request["args"], json!({}));
+
+ let (_directory, socket, server) = serve_once(
+ json!({"ok":true,"result":{"structuredContent":{"windows":[]},"content":[],"isError":false}}),
+ );
+ let driver = CuaDriver::connect(Some(socket)).unwrap();
+ assert!(driver
+ .list_windows(ListWindowsInput {
+ pid: Some(42),
+ on_screen_only: Some(true)
+ })
+ .await
+ .unwrap()
+ .windows
+ .is_empty());
+ let request = server.join().unwrap();
+ assert_eq!(request["name"], "list_windows");
+ assert_eq!(request["args"], json!({"pid":42,"on_screen_only":true}));
+}
+
+#[tokio::test]
+#[cfg(unix)]
+async fn typed_click_returns_action_facts_and_flat_native_arguments() {
+ let (_directory, socket, server) = serve_once(json!({"ok":true,"result":{
+ "structuredContent":{"effect":"confirmed","route":"accessibility","delivery":{"mode":"background"},"evidence":[{"kind":"value_readback"}]},
+ "content":[],"isError":false
+ }}));
+ let driver = CuaDriver::connect(Some(socket)).unwrap();
+ let output: ActionResult = driver
+ .click(ClickInput {
+ target: ActionTarget::Window {
+ pid: 42,
+ window_id: 73,
+ },
+ position: ClickPosition::Coordinates { x: 10.5, y: 20.0 },
+ delivery_mode: InputDeliveryMode::Background,
+ session: Some("run-1".into()),
+ button: None,
+ count: None,
+ })
+ .await
+ .unwrap();
+ assert_eq!(output.effect, cua_driver_contract::ActionEffect::Confirmed);
+ let request = server.join().unwrap();
+ assert_eq!(request["name"], "click");
+ assert_eq!(
+ request["args"],
+ json!({"target":{"kind":"window","pid":42,"window_id":73},"x":10.5,"y":20.0,"delivery_mode":"background","session":"run-1"})
+ );
+}
+
+#[tokio::test]
+#[cfg(unix)]
+async fn typed_snapshot_request_preserves_target_and_session() {
+ let (_directory, socket, server) = serve_once(json!({"ok":true,"result":snapshot_envelope()}));
+ let driver = CuaDriver::connect(Some(socket)).unwrap();
+ let input: GetWindowStateInput = serde_json::from_value(json!({
+ "pid":42,"window_id":73,"session":"run-1","include_screenshot":true,"max_elements":10
+ }))
+ .unwrap();
+ let output = driver.get_window_state(input).await.unwrap();
+ assert_eq!(output.images.len(), 2);
+ let request = server.join().unwrap();
+ assert_eq!(request["name"], "get_window_state");
+ assert_eq!(
+ request["args"],
+ json!({"pid":42,"window_id":73,"session":"run-1","include_screenshot":true,"max_elements":10})
+ );
+}
+
+#[tokio::test]
+async fn invalid_typed_click_is_rejected_before_transport() {
+ let driver = CuaDriver::connect(Some("/nonexistent/typed-click-test.sock".into())).unwrap();
+ for (target, position, delivery_mode) in [
+ (
+ ActionTarget::Desktop {
+ display_id: "primary".into(),
+ },
+ ClickPosition::Coordinates { x: 1.0, y: 2.0 },
+ InputDeliveryMode::Background,
+ ),
+ (
+ ActionTarget::Window {
+ pid: 42,
+ window_id: 73,
+ },
+ ClickPosition::Coordinates {
+ x: f64::NAN,
+ y: 2.0,
+ },
+ InputDeliveryMode::Background,
+ ),
+ (
+ ActionTarget::Desktop {
+ display_id: "primary".into(),
+ },
+ ClickPosition::Element {
+ element_token: "s1:0".into(),
+ },
+ InputDeliveryMode::Foreground,
+ ),
+ ] {
+ let error = driver
+ .click(ClickInput {
+ target,
+ position,
+ delivery_mode,
+ session: None,
+ button: None,
+ count: None,
+ })
+ .await
+ .unwrap_err();
+ assert!(matches!(error, DriverError::InvalidArguments { tool, .. } if tool == "click"));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
index 6390c7ec8f..8ad195ba0f 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
@@ -27,6 +27,11 @@ use std::process::{Command, Stdio};
use std::thread;
use std::time::{Duration, Instant};
+use cua_driver_contract::{
+ ActionTarget, ClickInput, ClickPosition, GetWindowStateInput, InputDeliveryMode, ListAppsInput,
+ ListWindowsInput, WindowStateOutput,
+};
+use cua_driver_sdk::{CuaDriver, DriverError};
use cua_driver_testkit::ax::{element_index_by_id, element_index_containing};
use cua_driver_testkit::e2e::{
recording_evidence, shared_web_route, write_declaration_from_env, write_result_from_env,
@@ -774,6 +779,222 @@ fn delivered_observation() -> Observation {
Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())
}
+fn sdk_window_input(fixture: &Fixture) -> GetWindowStateInput {
+ GetWindowStateInput {
+ pid: fixture.pid,
+ window_id: fixture.wid,
+ session: None,
+ query: None,
+ include_accessibility_tree: Some(true),
+ include_screenshot: Some(false),
+ screenshot_out_file: None,
+ max_elements: None,
+ max_depth: None,
+ max_dimension: None,
+ }
+}
+
+fn sdk_click_token(state: &WindowStateOutput) -> String {
+ let elements = state.elements.as_ref().expect("typed AX elements");
+ // The fixture's aria-label is its accessible name. Native adapters need
+ // not preserve the DOM id or visible text in the rendered markdown.
+ [
+ "border-click-target",
+ "Click target (left / right / double)",
+ ]
+ .into_iter()
+ .find_map(|label| {
+ elements
+ .iter()
+ .find(|element| element.label.as_deref() == Some(label))
+ .and_then(|element| element.element_token.clone())
+ })
+ .unwrap_or_else(|| panic!("typed click target has no snapshot-bound token: {elements:?}"))
+}
+
+#[test]
+fn typed_click_target_uses_accessible_label_without_markdown_ids() {
+ for label in [
+ "border-click-target",
+ "Click target (left / right / double)",
+ ] {
+ let state: WindowStateOutput = serde_json::from_value(serde_json::json!({
+ "pid": 42,
+ "window_id": 73,
+ "tree_markdown": "[9] button \"border-click-target\"",
+ "elements": [{
+ "element_index": 9,
+ "role": "button",
+ "depth": 1,
+ "label": label,
+ "element_token": "snapshot-token"
+ }]
+ }))
+ .unwrap();
+ assert_eq!(sdk_click_token(&state), "snapshot-token");
+ }
+}
+
+fn sdk_background_click(fixture: &Fixture, token: String) -> ClickInput {
+ ClickInput {
+ target: ActionTarget::Window {
+ pid: fixture.pid,
+ window_id: fixture.wid,
+ },
+ position: ClickPosition::Element {
+ element_token: token,
+ },
+ delivery_mode: InputDeliveryMode::Background,
+ session: None,
+ button: None,
+ count: None,
+ }
+}
+
+fn run_typed_sdk_native_window(fixture: &mut Fixture) -> Observation {
+ let runtime = tokio::runtime::Runtime::new().expect("SDK test runtime");
+ // The macOS harness authorizes the installed daemon, not cargo's test
+ // executable. Exercise the typed SDK over that verified native backend.
+ #[cfg(target_os = "macos")]
+ let sdk = {
+ let socket = std::env::var("CUA_E2E_MACOS_DAEMON_SOCKET")
+ .expect("canonical macOS harness must specify its authorized daemon socket");
+ CuaDriver::connect(Some(socket)).expect("connect SDK to authorized macOS daemon")
+ };
+ #[cfg(not(target_os = "macos"))]
+ let sdk = {
+ use cua_driver_sdk::{
+ ConfiguredDriverOptions, RuntimeAuthorizationOptions, SessionPermissionMode,
+ };
+
+ // Match the disposable desktop opt-in that McpDriver applies only to
+ // its child process, without changing this test process's environment.
+ if std::env::var_os("CUA_E2E_UNRESTRICTED_GUI").is_some() {
+ CuaDriver::create_configured(ConfiguredDriverOptions {
+ claude_code_compatibility: false,
+ authorization: RuntimeAuthorizationOptions {
+ allowed_modes: vec![SessionPermissionMode::Unrestricted],
+ compatibility_mode: SessionPermissionMode::Unrestricted,
+ compatibility_capability_manifest_path: None,
+ compatibility_bounded_manifest_path: None,
+ unrestricted_acknowledged: true,
+ max_session_ttl_seconds: 300,
+ max_idle_ttl_seconds: 300,
+ },
+ })
+ } else {
+ CuaDriver::create(None)
+ }
+ .expect("create in-process native SDK runtime")
+ };
+
+ runtime.block_on(async {
+ let apps = sdk
+ .list_apps(ListAppsInput {})
+ .await
+ .expect("typed app discovery");
+ assert!(
+ apps.apps
+ .iter()
+ .any(|app| app.pid == fixture.pid && app.running),
+ "typed app discovery must include the running fixture"
+ );
+ let windows = sdk
+ .list_windows(ListWindowsInput {
+ pid: None,
+ on_screen_only: None,
+ })
+ .await
+ .expect("typed window discovery");
+ assert!(
+ windows
+ .windows
+ .iter()
+ .any(|window| window.window_id == fixture.wid && window.pid == Some(fixture.pid)),
+ "typed window discovery must preserve exact fixture identity"
+ );
+
+ let before = sdk
+ .get_window_state(sdk_window_input(fixture))
+ .await
+ .expect("typed initial window state");
+ assert_eq!((before.pid, before.window_id), (fixture.pid, fixture.wid));
+ let stale_token = sdk_click_token(&before);
+ let current = sdk
+ .get_window_state(sdk_window_input(fixture))
+ .await
+ .expect("typed fresh window state invalidates previous token");
+ let current_token = sdk_click_token(&current);
+ assert_ne!(
+ stale_token, current_token,
+ "fresh snapshot must mint fresh tokens"
+ );
+ let journal_before = fixture.journal.snapshot();
+ let stale = sdk
+ .click(sdk_background_click(fixture, stale_token))
+ .await
+ .expect_err("stale SDK token must refuse");
+ assert!(
+ matches!(stale, DriverError::Tool { ref tool, ref error_code, .. }
+ if tool == "click" && error_code == "stale_element_token"),
+ "expected typed stale-token refusal, got {stale:?}"
+ );
+ thread::sleep(Duration::from_millis(150));
+ assert_eq!(
+ fixture.journal.snapshot(),
+ journal_before,
+ "stale SDK token changed fixture state"
+ );
+
+ let other_window = windows
+ .windows
+ .iter()
+ .find(|window| window.window_id != fixture.wid && window.pid != Some(fixture.pid))
+ .expect("foreground sentinel supplies a different native window");
+ let mut mismatched = sdk_background_click(fixture, current_token.clone());
+ mismatched.target = ActionTarget::Window {
+ pid: fixture.pid,
+ window_id: other_window.window_id,
+ };
+ let mismatch = sdk
+ .click(mismatched)
+ .await
+ .expect_err("SDK token must not address a different window");
+ assert!(
+ matches!(mismatch, DriverError::Tool { ref tool, ref error_code, .. }
+ if tool == "click" && matches!(error_code.as_str(),
+ "conflicting_element_target" | "window_target_not_found" | "window_target_mismatch")),
+ "expected typed window-identity refusal, got {mismatch:?}"
+ );
+ thread::sleep(Duration::from_millis(150));
+ assert_eq!(
+ fixture.journal.snapshot(),
+ journal_before,
+ "mismatched SDK window identity changed fixture state"
+ );
+
+ sdk.click(sdk_background_click(fixture, current_token.clone()))
+ .await
+ .expect("typed background element click");
+ assert_fixture_contains(fixture, "last_action=left_click");
+ // Match the fixture's delivery contract: its DOM journal is the effect
+ // oracle because embedded browsers can retain stale accessible text.
+ // A fresh native observation must still preserve identity and tokens.
+ let after = sdk
+ .get_window_state(sdk_window_input(fixture))
+ .await
+ .expect("typed post-action window state");
+ assert_eq!((after.pid, after.window_id), (fixture.pid, fixture.wid));
+ assert_ne!(
+ sdk_click_token(&after),
+ current_token,
+ "post-action observation must mint a fresh snapshot-bound token"
+ );
+ sdk.shutdown().await.expect("shut down typed SDK client");
+ });
+ delivered_observation()
+}
+
fn browser_ref_by_label(snapshot: &ToolResponse, label_fragment: &str) -> String {
snapshot.structured()["refs"]
.as_array()
@@ -1904,6 +2125,15 @@ fn shared_web_action_matrix_is_state_verified() {
let mut failure = None;
let mut selected = 0usize;
for spec in host_specs() {
+ let mut sdk_case = shared_case(&spec, "left_click", "ax", "background");
+ sdk_case.cell_id.push_str("-typed-sdk");
+ if cell_selected(&sdk_case) {
+ selected += 1;
+ let result = run_host_case_with_outcome(sdk_case, &spec, run_typed_sdk_native_window);
+ if failure.is_none() {
+ failure = result;
+ }
+ }
for (action, tool, marker) in [
("left_click", "click", "last_action=left_click"),
("right_click", "right_click", "last_action=right_click"),
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 7d8a240a76..bfb65c55c2 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -2,14 +2,13 @@
use async_trait::async_trait;
use cua_driver_contract::{
- ClickButton, ClickInput, DragInput, GetCursorPositionInput, GetDesktopStateInput,
- GetScreenSizeInput, HotkeyInput, InvokeMenuInput, MoveCursorInput, PressKeyInput, ScrollInput,
- TypeTextInput,
+ ClickButton, DragInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput,
+ HotkeyInput, InvokeMenuInput, MoveCursorInput, PressKeyInput, ScrollInput, TypeTextInput,
};
use cua_driver_core::{
protocol::ToolResult,
tool::{Tool, ToolDef, ToolRegistry},
- tool_args::{parse_typed_input, parse_typed_projection, ArgsExt},
+ tool_args::{parse_legacy_click_input, parse_typed_input, parse_typed_projection, ArgsExt},
window_target::{PidOnlyWindowTargetGuard, WindowTargetCandidate, WindowTargetCandidates},
};
use serde_json::{json, Value};
@@ -701,7 +700,7 @@ impl Tool for GetWindowStateTool {
"max_depth":{"type":"integer","minimum":1,"description":"Cap on the AT-SPI tree walk depth. Omit for the default (uncapped). Lower for deeply nested apps."},
"max_dimension":{"type":"integer","minimum":1,"description":"Optional cap on the returned screenshot's long edge, in pixels (aspect ratio preserved) — the cheap path for a small preview. Applied on top of the configured max_image_dimension ceiling; the tighter wins. Omit for the configured default."}
},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true, destructive: false, idempotent: false, open_world: false,
})
}
@@ -3328,7 +3327,7 @@ impl Tool for ClickTool {
"suggestion": "pass scope=desktop",
}));
}
- let input = match parse_typed_projection::<ClickInput>("click", &args) {
+ let input = match parse_legacy_click_input(&args) {
Ok(input) => input,
Err(result) => return result,
};
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index 7baa571611..dd4dd4ac68 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -14,11 +14,11 @@
//! to full-window space using the most recent `zoom` context stored per-pid.
use async_trait::async_trait;
-use cua_driver_contract::{ClickButton, ClickInput};
+use cua_driver_contract::ClickButton;
use cua_driver_core::{
protocol::ToolResult,
tool::{Tool, ToolDef},
- tool_args::parse_typed_projection,
+ tool_args::parse_legacy_click_input,
};
use serde_json::Value;
use std::sync::Arc;
@@ -245,7 +245,7 @@ impl Tool for ClickTool {
"suggestion": "pass scope=\"desktop\"",
}));
}
- let input = match parse_typed_projection::<ClickInput>("click", &args) {
+ let input = match parse_legacy_click_input(&args) {
Ok(input) => input,
Err(result) => return result,
};
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index db3d61c3bb..7862f183cf 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -317,14 +317,14 @@ async fn track_overlay_drag(
crate::overlay::send_command(key, cursor_overlay::OverlayCommand::SetPressed(false));
}
use cua_driver_contract::{
- ClickButton, ClickInput, DragInput, GetCursorPositionInput, GetDesktopStateInput,
- GetScreenSizeInput, HotkeyInput, InvokeMenuInput, MoveCursorInput, PressKeyInput,
- ScrollDirection, ScrollInput, TypeTextInput,
+ ClickButton, DragInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput,
+ HotkeyInput, InvokeMenuInput, MoveCursorInput, PressKeyInput, ScrollDirection, ScrollInput,
+ TypeTextInput,
};
use cua_driver_core::{
protocol::ToolResult,
tool::{Tool, ToolDef, ToolRegistry},
- tool_args::{parse_typed_input, parse_typed_projection},
+ tool_args::{parse_legacy_click_input, parse_typed_input, parse_typed_projection},
window_target::{PidOnlyWindowTargetGuard, WindowTargetCandidate, WindowTargetCandidates},
};
use serde_json::{json, Value};
@@ -3156,7 +3156,7 @@ impl Tool for ClickTool {
"suggestion": "pass scope=desktop",
}));
}
- let input = match parse_typed_projection::<ClickInput>("click", &args) {
+ let input = match parse_legacy_click_input(&args) {
Ok(input) => input,
Err(result) => return result,
};
diff --git a/libs/cua-driver/typescript/README.md b/libs/cua-driver/typescript/README.md
index 448843c716..c5546577ae 100644
--- a/libs/cua-driver/typescript/README.md
+++ b/libs/cua-driver/typescript/README.md
@@ -60,13 +60,15 @@ try {
} finally {
await driver.endSession(EndSessionInput.new({ session: "demo" }))
await driver.shutdown()
- driver.uniffiDestroy()
+ if ('uniffiDestroy' in driver && typeof driver.uniffiDestroy === 'function') {
+ driver.uniffiDestroy()
+ }
}
```
SDK operations are asynchronous and require a native library matching the host
OS and architecture. The macOS native package requires macOS 13 or newer.
-Desktop calls return a typed `ToolResult` with text,
+Desktop observations return a typed `ToolResult` with text,
images, verification/error metadata, and `structuredJson` / `rawJson` for
platform-extensible results. Session lifecycle calls return dedicated generated
records.
@@ -84,6 +86,30 @@ and installed with the local CLI; SDK and MCP tools select only an installed
theme ID. The built-in cursor shows the sanitized public session name in a
badge below the pointer.
+## Typed native-window migration
+
+The next breaking release adds typed app and window discovery, window snapshots,
+and token-based clicks. Version 0.25 supports native-window operations through
+the generic tool surface; it does not expose this typed window API. Upgrade the
+bindings and native library together.
+
+`listApps` returns `ListAppsOutput`, `listWindows` returns `ListWindowsOutput`,
+and `getWindowState` returns `WindowStateOutput`. `click` takes a required exact
+target, a coordinate or element-token position, and an explicit delivery mode.
+It returns `ActionResult` directly and raises `DriverError.Tool` on refusal.
+Other action methods retain `ToolResult`.
+
+Select a unique app and window, resolve an element from a fresh snapshot, request
+background delivery explicitly, then capture again to verify the intended UI
+change. An unsupported background route must not trigger an automatic foreground
+retry. Refresh stale tokens from the same exact window.
+
+Window IDs are `bigint`; do not convert them to `number`.
+
+See the [migration guide](../docs/native-window-sdk-migration.md) for input and
+return-type changes, and the [complete Python and TypeScript examples](https://cua.ai/docs/how-to-guides/driver/use-sdk-in-process)
+for discovery, token selection, verification, and shutdown.
+
## Authorization integrations
`standard` is promptless for normal automation. An application that needs to
@@ -130,7 +156,9 @@ try {
try {
// Application calls use driver; an agent runtime uses connection.mcp.
} finally {
- driver.uniffiDestroy()
+ if ('uniffiDestroy' in driver && typeof driver.uniffiDestroy === 'function') {
+ driver.uniffiDestroy()
+ }
}
} finally {
await embedded.stop()
diff --git a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts
index c2380cecc2..2c57cd782d 100644
--- a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts
+++ b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts
@@ -415,6 +415,94 @@ const FfiConverterTypeActionResult = (() => {
return new FFIConverter();
})();
+const stringConverter = (() => {
+ const encoder = new TextEncoder();
+ const decoder = new TextDecoder();
+ return {
+ stringToBytes: (s: string) => encoder.encode(s),
+ bytesToString: (ab: UniffiByteArray) => decoder.decode(ab),
+ stringByteLength: (s: string) => encoder.encode(s).byteLength,
+ writeStringIntoBuffer: (s: string, buf: any, offset: number): number => {
+ const view = new Uint8Array(
+ buf.arrayBuffer,
+ offset,
+ buf.arrayBuffer.byteLength - offset,
+ );
+ return encoder.encodeInto(s, view).written;
+ },
+ readStringFromBuffer: (buf: any, offset: number, length: number): string =>
+ decoder.decode(new Uint8Array(buf.arrayBuffer, offset, length)),
+ };
+})();
+const FfiConverterString = uniffiCreateFfiConverterString(stringConverter);
+
+export type AppInfo = {
+ pid: number,
+ name: string,
+ running: boolean,
+ active: boolean,
+ bundleId?: string,
+ launchPath?: string,
+ kind?: string,
+ lastUsed?: string
+}
+
+/**
+ * Generated factory for {@link AppInfo} record objects.
+ */
+export const AppInfo = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<AppInfo, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<AppInfo>,
+ });
+})();
+
+const FfiConverterTypeAppInfo = (() => {
+ type TypeName = AppInfo;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ pid: FfiConverterUInt32.read(from),
+ name: FfiConverterString.read(from),
+ running: FfiConverterBool.read(from),
+ active: FfiConverterBool.read(from),
+ bundleId: FfiConverterOptionalString.read(from),
+ launchPath: FfiConverterOptionalString.read(from),
+ kind: FfiConverterOptionalString.read(from),
+ lastUsed: FfiConverterOptionalString.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterUInt32.write(value.pid, into);
+ FfiConverterString.write(value.name, into);
+ FfiConverterBool.write(value.running, into);
+ FfiConverterBool.write(value.active, into);
+ FfiConverterOptionalString.write(value.bundleId, into);
+ FfiConverterOptionalString.write(value.launchPath, into);
+ FfiConverterOptionalString.write(value.kind, into);
+ FfiConverterOptionalString.write(value.lastUsed, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterUInt32.allocationSize(value.pid) +
+ FfiConverterString.allocationSize(value.name) +
+ FfiConverterBool.allocationSize(value.running) +
+ FfiConverterBool.allocationSize(value.active) +
+ FfiConverterOptionalString.allocationSize(value.bundleId) +
+ FfiConverterOptionalString.allocationSize(value.launchPath) +
+ FfiConverterOptionalString.allocationSize(value.kind) +
+ FfiConverterOptionalString.allocationSize(value.lastUsed);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
export type BoundsExpectation = {
x: number,
y: number,
@@ -470,27 +558,6 @@ const FfiConverterTypeBoundsExpectation = (() => {
return new FFIConverter();
})();
-const stringConverter = (() => {
- const encoder = new TextEncoder();
- const decoder = new TextDecoder();
- return {
- stringToBytes: (s: string) => encoder.encode(s),
- bytesToString: (ab: UniffiByteArray) => decoder.decode(ab),
- stringByteLength: (s: string) => encoder.encode(s).byteLength,
- writeStringIntoBuffer: (s: string, buf: any, offset: number): number => {
- const view = new Uint8Array(
- buf.arrayBuffer,
- offset,
- buf.arrayBuffer.byteLength - offset,
- );
- return encoder.encodeInto(s, view).written;
- },
- readStringFromBuffer: (buf: any, offset: number, length: number): string =>
- decoder.decode(new Uint8Array(buf.arrayBuffer, offset, length)),
- };
-})();
-const FfiConverterString = uniffiCreateFfiConverterString(stringConverter);
-
// Enum: ActionTarget
export enum ActionTarget_Tags {
@@ -644,23 +711,165 @@ const FfiConverterTypeActionTarget = (() => {
return new FFIConverter();
})();
-export enum DesktopScope {
- Desktop
+
+// Enum: ClickPosition
+export enum ClickPosition_Tags {
+ Coordinates = "Coordinates",
+ Element = "Element"
}
+export const ClickPosition = (() => {
-const FfiConverterTypeDesktopScope = (() => {
+ type Coordinates__interface = {
+ tag: ClickPosition_Tags.Coordinates;
+ inner:
+Readonly<{x: number; y: number}>
+ };
+ class Coordinates_ extends UniffiEnum implements Coordinates__interface {
+ /**
+ * @private
+ * This field is private and should not be used, use `tag` instead.
+ */
+ readonly [uniffiTypeNameSymbol] = "ClickPosition";
+ readonly tag = ClickPosition_Tags.Coordinates;
+ readonly inner:
+Readonly<{x: number; y: number}>;
+ constructor(
+inner: {x: number; y: number }) {
+ super("ClickPosition", "Coordinates");
+
+ this.inner = Object.freeze(inner);
+ }
+ static new(
+inner: {x: number; y: number }): Coordinates_ {
+ return new Coordinates_(inner);
+ }
+
+ static instanceOf(obj: any): obj is Coordinates_ {
+ return obj.tag === ClickPosition_Tags.Coordinates;
+ }
+
+ }
+
+ type Element__interface = {
+ tag: ClickPosition_Tags.Element;
+ inner:
+Readonly<{elementToken: string}>
+ };
+ class Element_ extends UniffiEnum implements Element__interface {
+ /**
+ * @private
+ * This field is private and should not be used, use `tag` instead.
+ */
+ readonly [uniffiTypeNameSymbol] = "ClickPosition";
+ readonly tag = ClickPosition_Tags.Element;
+ readonly inner:
+Readonly<{elementToken: string}>;
+ constructor(
+inner: {elementToken: string }) {
+ super("ClickPosition", "Element");
+
+ this.inner = Object.freeze(inner);
+ }
+ static new(
+inner: {elementToken: string }): Element_ {
+ return new Element_(inner);
+ }
+
+ static instanceOf(obj: any): obj is Element_ {
+ return obj.tag === ClickPosition_Tags.Element;
+ }
+
+ }
+
+ function instanceOf(obj: any): obj is ClickPosition {
+ return obj[uniffiTypeNameSymbol] === "ClickPosition";
+ }
+
+ return Object.freeze({
+ instanceOf,
+ Coordinates: Coordinates_,
+ Element: Element_
+ });
+
+})();
+export type ClickPosition = InstanceType<
+ typeof ClickPosition['Coordinates' | 'Element']
+>;
+
+// FfiConverter for enum ClickPosition
+const FfiConverterTypeClickPosition = (() => {
const ordinalConverter = FfiConverterInt32;
- type TypeName = DesktopScope;
+ type TypeName = ClickPosition;
class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
read(from: RustBuffer): TypeName {
switch (ordinalConverter.read(from)) {
- case 1: return DesktopScope.Desktop;
+ case 1: return new ClickPosition.Coordinates({x: FfiConverterFloat64.read(from), y: FfiConverterFloat64.read(from) });
+ case 2: return new ClickPosition.Element({elementToken: FfiConverterString.read(from) });
+ default: throw new UniffiInternalError.UnexpectedEnumCase();
+ }
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ switch (value.tag) {
+ case ClickPosition_Tags.Coordinates: {
+ ordinalConverter.write(1, into);
+ const inner = value.inner;
+ FfiConverterFloat64.write(inner.x, into);
+ FfiConverterFloat64.write(inner.y, into);
+ return;
+ }
+ case ClickPosition_Tags.Element: {
+ ordinalConverter.write(2, into);
+ const inner = value.inner;
+ FfiConverterString.write(inner.elementToken, into);
+ return;
+ }
+ default:
+ // Throwing from here means that ClickPosition_Tags hasn't matched an ordinal.
+ throw new UniffiInternalError.UnexpectedEnumCase();
+ }
+ }
+ allocationSize(value: TypeName): number {
+ switch (value.tag) {
+ case ClickPosition_Tags.Coordinates: {
+ const inner = value.inner;
+ let size = ordinalConverter.allocationSize(1);
+ size += FfiConverterFloat64.allocationSize(inner.x);
+ size += FfiConverterFloat64.allocationSize(inner.y);
+ return size;
+ }
+ case ClickPosition_Tags.Element: {
+ const inner = value.inner;
+ let size = ordinalConverter.allocationSize(2);
+ size += FfiConverterString.allocationSize(inner.elementToken);
+ return size;
+ }
+ default: throw new UniffiInternalError.UnexpectedEnumCase();
+ }
+ }
+ }
+ return new FFIConverter();
+})();
+
+export enum InputDeliveryMode {
+ Background,
+ Foreground
+}
+
+const FfiConverterTypeInputDeliveryMode = (() => {
+ const ordinalConverter = FfiConverterInt32;
+ type TypeName = InputDeliveryMode;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ switch (ordinalConverter.read(from)) {
+ case 1: return InputDeliveryMode.Background;
+ case 2: return InputDeliveryMode.Foreground;
default: throw new UniffiInternalError.UnexpectedEnumCase();
}
}
write(value: TypeName, into: RustBuffer): void {
switch (value) {
- case DesktopScope.Desktop: return ordinalConverter.write(1, into);
+ case InputDeliveryMode.Background: return ordinalConverter.write(1, into);
+ case InputDeliveryMode.Foreground: return ordinalConverter.write(2, into);
}
}
allocationSize(value: TypeName): number {
@@ -703,17 +912,9 @@ const FfiConverterTypeClickButton = (() => {
})();
export type ClickInput = {
- x: number,
- y: number,
- target?: ActionTarget,
- /**
- * Deprecated flat desktop target retained for wire compatibility.
- */
- scope?: DesktopScope,
- /**
- * For multi-call work, prefer a short public session label and repeat it on every call that
- * accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
- */
+ target: ActionTarget,
+ position: ClickPosition,
+ deliveryMode: InputDeliveryMode,
session?: string,
button?: ClickButton,
count?: number
@@ -740,29 +941,26 @@ const FfiConverterTypeClickInput = (() => {
class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
read(from: RustBuffer): TypeName {
return {
- x: FfiConverterFloat64.read(from),
- y: FfiConverterFloat64.read(from),
- target: FfiConverterOptionalTypeActionTarget.read(from),
- scope: FfiConverterOptionalTypeDesktopScope.read(from),
+ target: FfiConverterTypeActionTarget.read(from),
+ position: FfiConverterTypeClickPosition.read(from),
+ deliveryMode: FfiConverterTypeInputDeliveryMode.read(from),
session: FfiConverterOptionalString.read(from),
button: FfiConverterOptionalTypeClickButton.read(from),
count: FfiConverterOptionalUInt32.read(from)
};
}
write(value: TypeName, into: RustBuffer): void {
- FfiConverterFloat64.write(value.x, into);
- FfiConverterFloat64.write(value.y, into);
- FfiConverterOptionalTypeActionTarget.write(value.target, into);
- FfiConverterOptionalTypeDesktopScope.write(value.scope, into);
+ FfiConverterTypeActionTarget.write(value.target, into);
+ FfiConverterTypeClickPosition.write(value.position, into);
+ FfiConverterTypeInputDeliveryMode.write(value.deliveryMode, into);
FfiConverterOptionalString.write(value.session, into);
FfiConverterOptionalTypeClickButton.write(value.button, into);
FfiConverterOptionalUInt32.write(value.count, into);
}
allocationSize(value: TypeName): number {
- return FfiConverterFloat64.allocationSize(value.x) +
- FfiConverterFloat64.allocationSize(value.y) +
- FfiConverterOptionalTypeActionTarget.allocationSize(value.target) +
- FfiConverterOptionalTypeDesktopScope.allocationSize(value.scope) +
+ return FfiConverterTypeActionTarget.allocationSize(value.target) +
+ FfiConverterTypeClickPosition.allocationSize(value.position) +
+ FfiConverterTypeInputDeliveryMode.allocationSize(value.deliveryMode) +
FfiConverterOptionalString.allocationSize(value.session) +
FfiConverterOptionalTypeClickButton.allocationSize(value.button) +
FfiConverterOptionalUInt32.allocationSize(value.count);
@@ -1359,6 +1557,32 @@ const FfiConverterTypeCursorVisualOutput = (() => {
return new FFIConverter();
})();
+export enum DesktopScope {
+ Desktop
+}
+
+const FfiConverterTypeDesktopScope = (() => {
+ const ordinalConverter = FfiConverterInt32;
+ type TypeName = DesktopScope;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ switch (ordinalConverter.read(from)) {
+ case 1: return DesktopScope.Desktop;
+ default: throw new UniffiInternalError.UnexpectedEnumCase();
+ }
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ switch (value) {
+ case DesktopScope.Desktop: return ordinalConverter.write(1, into);
+ }
+ }
+ allocationSize(value: TypeName): number {
+ return ordinalConverter.allocationSize(0);
+ }
+ }
+ return new FFIConverter();
+})();
+
export type DragInput = {
fromX: number,
fromY: number,
@@ -1445,6 +1669,57 @@ const FfiConverterTypeDragInput = (() => {
return new FFIConverter();
})();
+export type ElementFrame = {
+ x: number,
+ y: number,
+ w: number,
+ h: number
+}
+
+/**
+ * Generated factory for {@link ElementFrame} record objects.
+ */
+export const ElementFrame = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<ElementFrame, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<ElementFrame>,
+ });
+})();
+
+const FfiConverterTypeElementFrame = (() => {
+ type TypeName = ElementFrame;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ x: FfiConverterFloat64.read(from),
+ y: FfiConverterFloat64.read(from),
+ w: FfiConverterFloat64.read(from),
+ h: FfiConverterFloat64.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterFloat64.write(value.x, into);
+ FfiConverterFloat64.write(value.y, into);
+ FfiConverterFloat64.write(value.w, into);
+ FfiConverterFloat64.write(value.h, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterFloat64.allocationSize(value.x) +
+ FfiConverterFloat64.allocationSize(value.y) +
+ FfiConverterFloat64.allocationSize(value.w) +
+ FfiConverterFloat64.allocationSize(value.h);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
export type ElementSelector = {
role?: string,
labelContains?: string
@@ -2047,6 +2322,81 @@ const FfiConverterTypeGetSessionStateInput = (() => {
return new FFIConverter();
})();
+export type GetWindowStateInput = {
+ pid: number,
+ windowId: bigint,
+ session?: string,
+ query?: string,
+ includeAccessibilityTree?: boolean,
+ includeScreenshot?: boolean,
+ screenshotOutFile?: string,
+ maxElements?: number,
+ maxDepth?: number,
+ maxDimension?: number
+}
+
+/**
+ * Generated factory for {@link GetWindowStateInput} record objects.
+ */
+export const GetWindowStateInput = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<GetWindowStateInput, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<GetWindowStateInput>,
+ });
+})();
+
+const FfiConverterTypeGetWindowStateInput = (() => {
+ type TypeName = GetWindowStateInput;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ pid: FfiConverterUInt32.read(from),
+ windowId: FfiConverterUInt64.read(from),
+ session: FfiConverterOptionalString.read(from),
+ query: FfiConverterOptionalString.read(from),
+ includeAccessibilityTree: FfiConverterOptionalBoolean.read(from),
+ includeScreenshot: FfiConverterOptionalBoolean.read(from),
+ screenshotOutFile: FfiConverterOptionalString.read(from),
+ maxElements: FfiConverterOptionalUInt32.read(from),
+ maxDepth: FfiConverterOptionalUInt32.read(from),
+ maxDimension: FfiConverterOptionalUInt32.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterUInt32.write(value.pid, into);
+ FfiConverterUInt64.write(value.windowId, into);
+ FfiConverterOptionalString.write(value.session, into);
+ FfiConverterOptionalString.write(value.query, into);
+ FfiConverterOptionalBoolean.write(value.includeAccessibilityTree, into);
+ FfiConverterOptionalBoolean.write(value.includeScreenshot, into);
+ FfiConverterOptionalString.write(value.screenshotOutFile, into);
+ FfiConverterOptionalUInt32.write(value.maxElements, into);
+ FfiConverterOptionalUInt32.write(value.maxDepth, into);
+ FfiConverterOptionalUInt32.write(value.maxDimension, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterUInt32.allocationSize(value.pid) +
+ FfiConverterUInt64.allocationSize(value.windowId) +
+ FfiConverterOptionalString.allocationSize(value.session) +
+ FfiConverterOptionalString.allocationSize(value.query) +
+ FfiConverterOptionalBoolean.allocationSize(value.includeAccessibilityTree) +
+ FfiConverterOptionalBoolean.allocationSize(value.includeScreenshot) +
+ FfiConverterOptionalString.allocationSize(value.screenshotOutFile) +
+ FfiConverterOptionalUInt32.allocationSize(value.maxElements) +
+ FfiConverterOptionalUInt32.allocationSize(value.maxDepth) +
+ FfiConverterOptionalUInt32.allocationSize(value.maxDimension);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
export type HotkeyInput = {
keys: Array<string>,
target?: ActionTarget,
@@ -2165,6 +2515,80 @@ const FfiConverterTypeInvokeMenuInput = (() => {
return new FFIConverter();
})();
+export type ListAppsInput = {
+}
+
+/**
+ * Generated factory for {@link ListAppsInput} record objects.
+ */
+export const ListAppsInput = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<ListAppsInput, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<ListAppsInput>,
+ });
+})();
+
+const FfiConverterTypeListAppsInput = (() => {
+ type TypeName = ListAppsInput;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ }
+ allocationSize(value: TypeName): number {
+ return 0;
+ }
+ };
+ return new FFIConverter();
+})();
+
+export type ListAppsOutput = {
+ apps: Array<AppInfo>
+}
+
+/**
+ * Generated factory for {@link ListAppsOutput} record objects.
+ */
+export const ListAppsOutput = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<ListAppsOutput, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<ListAppsOutput>,
+ });
+})();
+
+const FfiConverterTypeListAppsOutput = (() => {
+ type TypeName = ListAppsOutput;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ apps: FfiConverterSequenceTypeAppInfo.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterSequenceTypeAppInfo.write(value.apps, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterSequenceTypeAppInfo.allocationSize(value.apps);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
export type ListSessionsInput = {
/**
* Maximum number of content-free summaries to return (default 50, max
@@ -2437,6 +2861,229 @@ const FfiConverterTypeListSessionsOutput = (() => {
return new FFIConverter();
})();
+export type ListWindowsInput = {
+ pid?: number,
+ onScreenOnly?: boolean
+}
+
+/**
+ * Generated factory for {@link ListWindowsInput} record objects.
+ */
+export const ListWindowsInput = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<ListWindowsInput, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<ListWindowsInput>,
+ });
+})();
+
+const FfiConverterTypeListWindowsInput = (() => {
+ type TypeName = ListWindowsInput;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ pid: FfiConverterOptionalUInt32.read(from),
+ onScreenOnly: FfiConverterOptionalBoolean.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterOptionalUInt32.write(value.pid, into);
+ FfiConverterOptionalBoolean.write(value.onScreenOnly, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterOptionalUInt32.allocationSize(value.pid) +
+ FfiConverterOptionalBoolean.allocationSize(value.onScreenOnly);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
+export type WindowBounds = {
+ x: number,
+ y: number,
+ width: number,
+ height: number
+}
+
+/**
+ * Generated factory for {@link WindowBounds} record objects.
+ */
+export const WindowBounds = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<WindowBounds, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<WindowBounds>,
+ });
+})();
+
+const FfiConverterTypeWindowBounds = (() => {
+ type TypeName = WindowBounds;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ x: FfiConverterFloat64.read(from),
+ y: FfiConverterFloat64.read(from),
+ width: FfiConverterFloat64.read(from),
+ height: FfiConverterFloat64.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterFloat64.write(value.x, into);
+ FfiConverterFloat64.write(value.y, into);
+ FfiConverterFloat64.write(value.width, into);
+ FfiConverterFloat64.write(value.height, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterFloat64.allocationSize(value.x) +
+ FfiConverterFloat64.allocationSize(value.y) +
+ FfiConverterFloat64.allocationSize(value.width) +
+ FfiConverterFloat64.allocationSize(value.height);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
+export type WindowInfo = {
+ windowId: bigint,
+ pid?: number,
+ appName: string,
+ title: string,
+ bounds: WindowBounds,
+ isOnScreen: boolean,
+ /**
+ * Higher values are closer to the front. Null means stacking order is unavailable; callers must not infer an order from array position.
+ */
+ zIndex?: bigint,
+ layer?: number,
+ minimized?: boolean,
+ currentSpaceId?: bigint,
+ onCurrentSpace?: boolean,
+ spaceIds?: Array<bigint>
+}
+
+/**
+ * Generated factory for {@link WindowInfo} record objects.
+ */
+export const WindowInfo = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<WindowInfo, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<WindowInfo>,
+ });
+})();
+
+const FfiConverterTypeWindowInfo = (() => {
+ type TypeName = WindowInfo;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ windowId: FfiConverterUInt64.read(from),
+ pid: FfiConverterOptionalUInt32.read(from),
+ appName: FfiConverterString.read(from),
+ title: FfiConverterString.read(from),
+ bounds: FfiConverterTypeWindowBounds.read(from),
+ isOnScreen: FfiConverterBool.read(from),
+ zIndex: FfiConverterOptionalInt64.read(from),
+ layer: FfiConverterOptionalInt32.read(from),
+ minimized: FfiConverterOptionalBoolean.read(from),
+ currentSpaceId: FfiConverterOptionalUInt64.read(from),
+ onCurrentSpace: FfiConverterOptionalBoolean.read(from),
+ spaceIds: FfiConverterOptionalSequenceUInt64.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterUInt64.write(value.windowId, into);
+ FfiConverterOptionalUInt32.write(value.pid, into);
+ FfiConverterString.write(value.appName, into);
+ FfiConverterString.write(value.title, into);
+ FfiConverterTypeWindowBounds.write(value.bounds, into);
+ FfiConverterBool.write(value.isOnScreen, into);
+ FfiConverterOptionalInt64.write(value.zIndex, into);
+ FfiConverterOptionalInt32.write(value.layer, into);
+ FfiConverterOptionalBoolean.write(value.minimized, into);
+ FfiConverterOptionalUInt64.write(value.currentSpaceId, into);
+ FfiConverterOptionalBoolean.write(value.onCurrentSpace, into);
+ FfiConverterOptionalSequenceUInt64.write(value.spaceIds, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterUInt64.allocationSize(value.windowId) +
+ FfiConverterOptionalUInt32.allocationSize(value.pid) +
+ FfiConverterString.allocationSize(value.appName) +
+ FfiConverterString.allocationSize(value.title) +
+ FfiConverterTypeWindowBounds.allocationSize(value.bounds) +
+ FfiConverterBool.allocationSize(value.isOnScreen) +
+ FfiConverterOptionalInt64.allocationSize(value.zIndex) +
+ FfiConverterOptionalInt32.allocationSize(value.layer) +
+ FfiConverterOptionalBoolean.allocationSize(value.minimized) +
+ FfiConverterOptionalUInt64.allocationSize(value.currentSpaceId) +
+ FfiConverterOptionalBoolean.allocationSize(value.onCurrentSpace) +
+ FfiConverterOptionalSequenceUInt64.allocationSize(value.spaceIds);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
+export type ListWindowsOutput = {
+ windows: Array<WindowInfo>,
+ currentSpaceId?: bigint
+}
+
+/**
+ * Generated factory for {@link ListWindowsOutput} record objects.
+ */
+export const ListWindowsOutput = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<ListWindowsOutput, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<ListWindowsOutput>,
+ });
+})();
+
+const FfiConverterTypeListWindowsOutput = (() => {
+ type TypeName = ListWindowsOutput;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ windows: FfiConverterSequenceTypeWindowInfo.read(from),
+ currentSpaceId: FfiConverterOptionalUInt64.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterSequenceTypeWindowInfo.write(value.windows, into);
+ FfiConverterOptionalUInt64.write(value.currentSpaceId, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterSequenceTypeWindowInfo.allocationSize(value.windows) +
+ FfiConverterOptionalUInt64.allocationSize(value.currentSpaceId);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
export type MoveCursorInput = {
x: number,
y: number,
@@ -3328,6 +3975,49 @@ const FfiConverterTypeSetWindowFrameInput = (() => {
return new FFIConverter();
})();
+export type SnapshotImage = {
+ mimeType: string,
+ dataBase64: string
+}
+
+/**
+ * Generated factory for {@link SnapshotImage} record objects.
+ */
+export const SnapshotImage = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<SnapshotImage, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<SnapshotImage>,
+ });
+})();
+
+const FfiConverterTypeSnapshotImage = (() => {
+ type TypeName = SnapshotImage;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ mimeType: FfiConverterString.read(from),
+ dataBase64: FfiConverterString.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterString.write(value.mimeType, into);
+ FfiConverterString.write(value.dataBase64, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterString.allocationSize(value.mimeType) +
+ FfiConverterString.allocationSize(value.dataBase64);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
export type StartSessionInput = {
/**
* Optional stable public label for this run (e.g. "research-run-1").
@@ -3724,6 +4414,235 @@ const FfiConverterTypeVerifyStateOutput = (() => {
return new FFIConverter();
})();
+export type WindowElement = {
+ elementIndex: bigint,
+ role: string,
+ depth: number,
+ elementToken?: string,
+ label?: string,
+ value?: string,
+ valueDescription?: string,
+ enabled?: boolean,
+ selected?: boolean,
+ inWebContent?: boolean,
+ actions?: Array<string>,
+ parentIndex?: bigint,
+ frame?: ElementFrame,
+ min?: number,
+ max?: number
+}
+
+/**
+ * Generated factory for {@link WindowElement} record objects.
+ */
+export const WindowElement = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<WindowElement, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<WindowElement>,
+ });
+})();
+
+const FfiConverterTypeWindowElement = (() => {
+ type TypeName = WindowElement;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ elementIndex: FfiConverterUInt64.read(from),
+ role: FfiConverterString.read(from),
+ depth: FfiConverterUInt32.read(from),
+ elementToken: FfiConverterOptionalString.read(from),
+ label: FfiConverterOptionalString.read(from),
+ value: FfiConverterOptionalString.read(from),
+ valueDescription: FfiConverterOptionalString.read(from),
+ enabled: FfiConverterOptionalBoolean.read(from),
+ selected: FfiConverterOptionalBoolean.read(from),
+ inWebContent: FfiConverterOptionalBoolean.read(from),
+ actions: FfiConverterOptionalSequenceString.read(from),
+ parentIndex: FfiConverterOptionalUInt64.read(from),
+ frame: FfiConverterOptionalTypeElementFrame.read(from),
+ min: FfiConverterOptionalFloat64.read(from),
+ max: FfiConverterOptionalFloat64.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterUInt64.write(value.elementIndex, into);
+ FfiConverterString.write(value.role, into);
+ FfiConverterUInt32.write(value.depth, into);
+ FfiConverterOptionalString.write(value.elementToken, into);
+ FfiConverterOptionalString.write(value.label, into);
+ FfiConverterOptionalString.write(value.value, into);
+ FfiConverterOptionalString.write(value.valueDescription, into);
+ FfiConverterOptionalBoolean.write(value.enabled, into);
+ FfiConverterOptionalBoolean.write(value.selected, into);
+ FfiConverterOptionalBoolean.write(value.inWebContent, into);
+ FfiConverterOptionalSequenceString.write(value.actions, into);
+ FfiConverterOptionalUInt64.write(value.parentIndex, into);
+ FfiConverterOptionalTypeElementFrame.write(value.frame, into);
+ FfiConverterOptionalFloat64.write(value.min, into);
+ FfiConverterOptionalFloat64.write(value.max, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterUInt64.allocationSize(value.elementIndex) +
+ FfiConverterString.allocationSize(value.role) +
+ FfiConverterUInt32.allocationSize(value.depth) +
+ FfiConverterOptionalString.allocationSize(value.elementToken) +
+ FfiConverterOptionalString.allocationSize(value.label) +
+ FfiConverterOptionalString.allocationSize(value.value) +
+ FfiConverterOptionalString.allocationSize(value.valueDescription) +
+ FfiConverterOptionalBoolean.allocationSize(value.enabled) +
+ FfiConverterOptionalBoolean.allocationSize(value.selected) +
+ FfiConverterOptionalBoolean.allocationSize(value.inWebContent) +
+ FfiConverterOptionalSequenceString.allocationSize(value.actions) +
+ FfiConverterOptionalUInt64.allocationSize(value.parentIndex) +
+ FfiConverterOptionalTypeElementFrame.allocationSize(value.frame) +
+ FfiConverterOptionalFloat64.allocationSize(value.min) +
+ FfiConverterOptionalFloat64.allocationSize(value.max);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
+export type WindowStateOutput = {
+ pid: number,
+ windowId: bigint,
+ snapshotId?: string,
+ appName?: string,
+ windowTitle?: string,
+ treeMarkdown?: string,
+ elements?: Array<WindowElement>,
+ elementCount?: bigint,
+ totalElementCount?: bigint,
+ returnedElementCount?: bigint,
+ filteredElementCount?: bigint,
+ elementsComplete?: boolean,
+ degraded?: boolean,
+ degradedReason?: string,
+ truncated?: boolean,
+ truncationReason?: string,
+ screenshotWidth?: number,
+ screenshotHeight?: number,
+ screenshotScale?: number,
+ screenshotMimeType?: string,
+ screenshotFilePath?: string,
+ screenshotFrameValid?: boolean,
+ windowBounds?: WindowBounds,
+ /**
+ * Image content belongs to the MCP envelope, never structuredContent.
+ */
+ images: Array<SnapshotImage>
+}
+
+/**
+ * Generated factory for {@link WindowStateOutput} record objects.
+ */
+export const WindowStateOutput = (() => {
+ const defaults = () => ({
+ });
+ const create = (() => {
+ return uniffiCreateRecord<WindowStateOutput, ReturnType<typeof defaults>>(defaults);
+ })();
+ return Object.freeze({
+ create,
+ new: create,
+ defaults: () => Object.freeze(defaults()) as Partial<WindowStateOutput>,
+ });
+})();
+
+const FfiConverterTypeWindowStateOutput = (() => {
+ type TypeName = WindowStateOutput;
+ class FFIConverter extends AbstractFfiConverterByteArray<TypeName> {
+ read(from: RustBuffer): TypeName {
+ return {
+ pid: FfiConverterUInt32.read(from),
+ windowId: FfiConverterUInt64.read(from),
+ snapshotId: FfiConverterOptionalString.read(from),
+ appName: FfiConverterOptionalString.read(from),
+ windowTitle: FfiConverterOptionalString.read(from),
+ treeMarkdown: FfiConverterOptionalString.read(from),
+ elements: FfiConverterOptionalSequenceTypeWindowElement.read(from),
+ elementCount: FfiConverterOptionalUInt64.read(from),
+ totalElementCount: FfiConverterOptionalUInt64.read(from),
+ returnedElementCount: FfiConverterOptionalUInt64.read(from),
+ filteredElementCount: FfiConverterOptionalUInt64.read(from),
+ elementsComplete: FfiConverterOptionalBoolean.read(from),
+ degraded: FfiConverterOptionalBoolean.read(from),
+ degradedReason: FfiConverterOptionalString.read(from),
+ truncated: FfiConverterOptionalBoolean.read(from),
+ truncationReason: FfiConverterOptionalString.read(from),
+ screenshotWidth: FfiConverterOptionalUInt32.read(from),
+ screenshotHeight: FfiConverterOptionalUInt32.read(from),
+ screenshotScale: FfiConverterOptionalFloat64.read(from),
+ screenshotMimeType: FfiConverterOptionalString.read(from),
+ screenshotFilePath: FfiConverterOptionalString.read(from),
+ screenshotFrameValid: FfiConverterOptionalBoolean.read(from),
+ windowBounds: FfiConverterOptionalTypeWindowBounds.read(from),
+ images: FfiConverterSequenceTypeSnapshotImage.read(from)
+ };
+ }
+ write(value: TypeName, into: RustBuffer): void {
+ FfiConverterUInt32.write(value.pid, into);
+ FfiConverterUInt64.write(value.windowId, into);
+ FfiConverterOptionalString.write(value.snapshotId, into);
+ FfiConverterOptionalString.write(value.appName, into);
+ FfiConverterOptionalString.write(value.windowTitle, into);
+ FfiConverterOptionalString.write(value.treeMarkdown, into);
+ FfiConverterOptionalSequenceTypeWindowElement.write(value.elements, into);
+ FfiConverterOptionalUInt64.write(value.elementCount, into);
+ FfiConverterOptionalUInt64.write(value.totalElementCount, into);
+ FfiConverterOptionalUInt64.write(value.returnedElementCount, into);
+ FfiConverterOptionalUInt64.write(value.filteredElementCount, into);
+ FfiConverterOptionalBoolean.write(value.elementsComplete, into);
+ FfiConverterOptionalBoolean.write(value.degraded, into);
+ FfiConverterOptionalString.write(value.degradedReason, into);
+ FfiConverterOptionalBoolean.write(value.truncated, into);
+ FfiConverterOptionalString.write(value.truncationReason, into);
+ FfiConverterOptionalUInt32.write(value.screenshotWidth, into);
+ FfiConverterOptionalUInt32.write(value.screenshotHeight, into);
+ FfiConverterOptionalFloat64.write(value.screenshotScale, into);
+ FfiConverterOptionalString.write(value.screenshotMimeType, into);
+ FfiConverterOptionalString.write(value.screenshotFilePath, into);
+ FfiConverterOptionalBoolean.write(value.screenshotFrameValid, into);
+ FfiConverterOptionalTypeWindowBounds.write(value.windowBounds, into);
+ FfiConverterSequenceTypeSnapshotImage.write(value.images, into);
+ }
+ allocationSize(value: TypeName): number {
+ return FfiConverterUInt32.allocationSize(value.pid) +
+ FfiConverterUInt64.allocationSize(value.windowId) +
+ FfiConverterOptionalString.allocationSize(value.snapshotId) +
+ FfiConverterOptionalString.allocationSize(value.appName) +
+ FfiConverterOptionalString.allocationSize(value.windowTitle) +
+ FfiConverterOptionalString.allocationSize(value.treeMarkdown) +
+ FfiConverterOptionalSequenceTypeWindowElement.allocationSize(value.elements) +
+ FfiConverterOptionalUInt64.allocationSize(value.elementCount) +
+ FfiConverterOptionalUInt64.allocationSize(value.totalElementCount) +
+ FfiConverterOptionalUInt64.allocationSize(value.returnedElementCount) +
+ FfiConverterOptionalUInt64.allocationSize(value.filteredElementCount) +
+ FfiConverterOptionalBoolean.allocationSize(value.elementsComplete) +
+ FfiConverterOptionalBoolean.allocationSize(value.degraded) +
+ FfiConverterOptionalString.allocationSize(value.degradedReason) +
+ FfiConverterOptionalBoolean.allocationSize(value.truncated) +
+ FfiConverterOptionalString.allocationSize(value.truncationReason) +
+ FfiConverterOptionalUInt32.allocationSize(value.screenshotWidth) +
+ FfiConverterOptionalUInt32.allocationSize(value.screenshotHeight) +
+ FfiConverterOptionalFloat64.allocationSize(value.screenshotScale) +
+ FfiConverterOptionalString.allocationSize(value.screenshotMimeType) +
+ FfiConverterOptionalString.allocationSize(value.screenshotFilePath) +
+ FfiConverterOptionalBoolean.allocationSize(value.screenshotFrameValid) +
+ FfiConverterOptionalTypeWindowBounds.allocationSize(value.windowBounds) +
+ FfiConverterSequenceTypeSnapshotImage.allocationSize(value.images);
+
+ }
+ };
+ return new FFIConverter();
+})();
+
export enum Platform {
Macos,
Windows,
@@ -3771,24 +4690,24 @@ const FfiConverterOptionalSequenceTypeActionEvidence = new FfiConverterOptional(
// FfiConverter for ActionEscalation | undefined
const FfiConverterOptionalTypeActionEscalation = new FfiConverterOptional(FfiConverterTypeActionEscalation);
-// FfiConverter for number | undefined
-const FfiConverterOptionalFloat64 = new FfiConverterOptional(FfiConverterFloat64);
-
-// FfiConverter for ActionTarget | undefined
-const FfiConverterOptionalTypeActionTarget = new FfiConverterOptional(FfiConverterTypeActionTarget);
-
-// FfiConverter for DesktopScope | undefined
-const FfiConverterOptionalTypeDesktopScope = new FfiConverterOptional(FfiConverterTypeDesktopScope);
-
// FfiConverter for string | undefined
const FfiConverterOptionalString = new FfiConverterOptional(FfiConverterString);
+// FfiConverter for number | undefined
+const FfiConverterOptionalFloat64 = new FfiConverterOptional(FfiConverterFloat64);
+
// FfiConverter for ClickButton | undefined
const FfiConverterOptionalTypeClickButton = new FfiConverterOptional(FfiConverterTypeClickButton);
// FfiConverter for Array<string>
const FfiConverterSequenceString = new FfiConverterArray(FfiConverterString);
+// FfiConverter for ActionTarget | undefined
+const FfiConverterOptionalTypeActionTarget = new FfiConverterOptional(FfiConverterTypeActionTarget);
+
+// FfiConverter for DesktopScope | undefined
+const FfiConverterOptionalTypeDesktopScope = new FfiConverterOptional(FfiConverterTypeDesktopScope);
+
// FfiConverter for bigint | undefined
const FfiConverterOptionalUInt64 = new FfiConverterOptional(FfiConverterUInt64);
@@ -3801,9 +4720,27 @@ const FfiConverterOptionalBoolean = new FfiConverterOptional(FfiConverterBool);
// FfiConverter for CursorPointOutput | undefined
const FfiConverterOptionalTypeCursorPointOutput = new FfiConverterOptional(FfiConverterTypeCursorPointOutput);
+// FfiConverter for Array<AppInfo>
+const FfiConverterSequenceTypeAppInfo = new FfiConverterArray(FfiConverterTypeAppInfo);
+
// FfiConverter for Array<SessionOutput>
const FfiConverterSequenceTypeSessionOutput = new FfiConverterArray(FfiConverterTypeSessionOutput);
+// FfiConverter for bigint | undefined
+const FfiConverterOptionalInt64 = new FfiConverterOptional(FfiConverterInt64);
+
+// FfiConverter for number | undefined
+const FfiConverterOptionalInt32 = new FfiConverterOptional(FfiConverterInt32);
+
+// FfiConverter for Array<bigint>
+const FfiConverterSequenceUInt64 = new FfiConverterArray(FfiConverterUInt64);
+
+// FfiConverter for Array<bigint> | undefined
+const FfiConverterOptionalSequenceUInt64 = new FfiConverterOptional(FfiConverterSequenceUInt64);
+
+// FfiConverter for Array<WindowInfo>
+const FfiConverterSequenceTypeWindowInfo = new FfiConverterArray(FfiConverterTypeWindowInfo);
+
// FfiConverter for UnknownReason | undefined
const FfiConverterOptionalTypeUnknownReason = new FfiConverterOptional(FfiConverterTypeUnknownReason);
@@ -3834,6 +4771,21 @@ const FfiConverterSequenceTypeStatePredicate = new FfiConverterArray(FfiConverte
// FfiConverter for Array<PredicateOutcome>
const FfiConverterSequenceTypePredicateOutcome = new FfiConverterArray(FfiConverterTypePredicateOutcome);
+// FfiConverter for ElementFrame | undefined
+const FfiConverterOptionalTypeElementFrame = new FfiConverterOptional(FfiConverterTypeElementFrame);
+
+// FfiConverter for Array<WindowElement>
+const FfiConverterSequenceTypeWindowElement = new FfiConverterArray(FfiConverterTypeWindowElement);
+
+// FfiConverter for Array<WindowElement> | undefined
+const FfiConverterOptionalSequenceTypeWindowElement = new FfiConverterOptional(FfiConverterSequenceTypeWindowElement);
+
+// FfiConverter for WindowBounds | undefined
+const FfiConverterOptionalTypeWindowBounds = new FfiConverterOptional(FfiConverterTypeWindowBounds);
+
+// FfiConverter for Array<SnapshotImage>
+const FfiConverterSequenceTypeSnapshotImage = new FfiConverterArray(FfiConverterTypeSnapshotImage);
+
/**
* This should be called before anything else.
@@ -3870,10 +4822,12 @@ export default Object.freeze({
FfiConverterTypeActionResult,
FfiConverterTypeActionRoute,
FfiConverterTypeActionTarget,
+ FfiConverterTypeAppInfo,
FfiConverterTypeBoundsExpectation,
FfiConverterTypeCaptureScope,
FfiConverterTypeClickButton,
FfiConverterTypeClickInput,
+ FfiConverterTypeClickPosition,
FfiConverterTypeClipboardReadInput,
FfiConverterTypeClipboardReadOutput,
FfiConverterTypeClipboardWriteInput,
@@ -3888,6 +4842,7 @@ export default Object.freeze({
FfiConverterTypeDesktopScope,
FfiConverterTypeDragInput,
FfiConverterTypeEffectiveScope,
+ FfiConverterTypeElementFrame,
FfiConverterTypeElementPredicate,
FfiConverterTypeElementSelector,
FfiConverterTypeEndSessionInput,
@@ -3901,10 +4856,16 @@ export default Object.freeze({
FfiConverterTypeGetScreenSizeInput,
FfiConverterTypeGetSessionInput,
FfiConverterTypeGetSessionStateInput,
+ FfiConverterTypeGetWindowStateInput,
FfiConverterTypeHotkeyInput,
+ FfiConverterTypeInputDeliveryMode,
FfiConverterTypeInvokeMenuInput,
+ FfiConverterTypeListAppsInput,
+ FfiConverterTypeListAppsOutput,
FfiConverterTypeListSessionsInput,
FfiConverterTypeListSessionsOutput,
+ FfiConverterTypeListWindowsInput,
+ FfiConverterTypeListWindowsOutput,
FfiConverterTypeMoveCursorInput,
FfiConverterTypePlatform,
FfiConverterTypePredicateOutcome,
@@ -3924,6 +4885,7 @@ export default Object.freeze({
FfiConverterTypeSetAgentCursorThemeInput,
FfiConverterTypeSetAgentCursorThemeOutput,
FfiConverterTypeSetWindowFrameInput,
+ FfiConverterTypeSnapshotImage,
FfiConverterTypeStartSessionInput,
FfiConverterTypeStartSessionOutput,
FfiConverterTypeStatePredicate,
@@ -3932,6 +4894,10 @@ export default Object.freeze({
FfiConverterTypeVerificationStatus,
FfiConverterTypeVerifyStateInput,
FfiConverterTypeVerifyStateOutput,
+ FfiConverterTypeWindowBounds,
+ FfiConverterTypeWindowElement,
+ FfiConverterTypeWindowInfo,
FfiConverterTypeWindowPredicate,
+ FfiConverterTypeWindowStateOutput,
}
});
diff --git a/libs/cua-driver/typescript/src/native/cua_driver_sdk-ffi.ts b/libs/cua-driver/typescript/src/native/cua_driver_sdk-ffi.ts
index 6e678242c8..3e37cab8fb 100644
--- a/libs/cua-driver/typescript/src/native/cua_driver_sdk-ffi.ts
+++ b/libs/cua-driver/typescript/src/native/cua_driver_sdk-ffi.ts
@@ -508,6 +508,11 @@ const DEFINITIONS = {
ret: FfiType.Handle,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_fn_method_cuadriver_get_window_state": {
+ args: [FfiType.Handle, FfiType.RustBuffer],
+ ret: FfiType.Handle,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_fn_method_cuadriver_hotkey": {
args: [FfiType.Handle, FfiType.RustBuffer],
ret: FfiType.Handle,
@@ -523,6 +528,11 @@ const DEFINITIONS = {
ret: FfiType.Int8,
hasRustCallStatus: true,
},
+ "uniffi_cua_driver_sdk_fn_method_cuadriver_list_apps": {
+ args: [FfiType.Handle, FfiType.RustBuffer],
+ ret: FfiType.Handle,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_fn_method_cuadriver_list_host_sessions_json": {
args: [FfiType.Handle],
ret: FfiType.Handle,
@@ -538,6 +548,11 @@ const DEFINITIONS = {
ret: FfiType.Handle,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_fn_method_cuadriver_list_windows": {
+ args: [FfiType.Handle, FfiType.RustBuffer],
+ ret: FfiType.Handle,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_fn_method_cuadriver_metadata": {
args: [FfiType.Handle],
ret: FfiType.Handle,
@@ -678,6 +693,11 @@ const DEFINITIONS = {
ret: FfiType.Handle,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_fn_method_cuadriversession_get_window_state": {
+ args: [FfiType.Handle, FfiType.RustBuffer],
+ ret: FfiType.Handle,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_fn_method_cuadriversession_hotkey": {
args: [FfiType.Handle, FfiType.RustBuffer],
ret: FfiType.Handle,
@@ -688,11 +708,21 @@ const DEFINITIONS = {
ret: FfiType.Handle,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_fn_method_cuadriversession_list_apps": {
+ args: [FfiType.Handle, FfiType.RustBuffer],
+ ret: FfiType.Handle,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_fn_method_cuadriversession_list_sessions": {
args: [FfiType.Handle, FfiType.RustBuffer],
ret: FfiType.Handle,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_fn_method_cuadriversession_list_windows": {
+ args: [FfiType.Handle, FfiType.RustBuffer],
+ ret: FfiType.Handle,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_fn_method_cuadriversession_move_cursor": {
args: [FfiType.Handle, FfiType.RustBuffer],
ret: FfiType.Handle,
@@ -998,6 +1028,11 @@ const DEFINITIONS = {
ret: FfiType.UInt16,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_checksum_method_cuadriver_get_window_state": {
+ args: [],
+ ret: FfiType.UInt16,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_checksum_method_cuadriver_hotkey": {
args: [],
ret: FfiType.UInt16,
@@ -1013,6 +1048,11 @@ const DEFINITIONS = {
ret: FfiType.UInt16,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_checksum_method_cuadriver_list_apps": {
+ args: [],
+ ret: FfiType.UInt16,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_checksum_method_cuadriver_list_host_sessions_json": {
args: [],
ret: FfiType.UInt16,
@@ -1028,6 +1068,11 @@ const DEFINITIONS = {
ret: FfiType.UInt16,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_checksum_method_cuadriver_list_windows": {
+ args: [],
+ ret: FfiType.UInt16,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_checksum_method_cuadriver_metadata": {
args: [],
ret: FfiType.UInt16,
@@ -1168,6 +1213,11 @@ const DEFINITIONS = {
ret: FfiType.UInt16,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_window_state": {
+ args: [],
+ ret: FfiType.UInt16,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_checksum_method_cuadriversession_hotkey": {
args: [],
ret: FfiType.UInt16,
@@ -1178,11 +1228,21 @@ const DEFINITIONS = {
ret: FfiType.UInt16,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_apps": {
+ args: [],
+ ret: FfiType.UInt16,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_sessions": {
args: [],
ret: FfiType.UInt16,
hasRustCallStatus: false,
},
+ "uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_windows": {
+ args: [],
+ ret: FfiType.UInt16,
+ hasRustCallStatus: false,
+ },
"uniffi_cua_driver_sdk_checksum_method_cuadriversession_move_cursor": {
args: [],
ret: FfiType.UInt16,
@@ -1547,12 +1607,15 @@ interface NativeModuleInterface {
uniffi_cua_driver_sdk_fn_method_cuadriver_get_screen_size(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_get_session(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_get_session_state(uniffiSelf: bigint, input: Uint8Array): bigint;
+ uniffi_cua_driver_sdk_fn_method_cuadriver_get_window_state(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_hotkey(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_invoke_menu(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_is_available(uniffiSelf: bigint, uniffi_out_err: UniffiRustCallStatus): number;
+ uniffi_cua_driver_sdk_fn_method_cuadriver_list_apps(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_list_host_sessions_json(uniffiSelf: bigint): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_list_sessions(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_list_tools_json(uniffiSelf: bigint): bigint;
+ uniffi_cua_driver_sdk_fn_method_cuadriver_list_windows(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_metadata(uniffiSelf: bigint): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_move_cursor(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriver_press_key(uniffiSelf: bigint, input: Uint8Array): bigint;
@@ -1581,9 +1644,12 @@ interface NativeModuleInterface {
uniffi_cua_driver_sdk_fn_method_cuadriversession_get_screen_size(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriversession_get_session(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriversession_get_session_state(uniffiSelf: bigint, input: Uint8Array): bigint;
+ uniffi_cua_driver_sdk_fn_method_cuadriversession_get_window_state(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriversession_hotkey(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriversession_invoke_menu(uniffiSelf: bigint, input: Uint8Array): bigint;
+ uniffi_cua_driver_sdk_fn_method_cuadriversession_list_apps(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriversession_list_sessions(uniffiSelf: bigint, input: Uint8Array): bigint;
+ uniffi_cua_driver_sdk_fn_method_cuadriversession_list_windows(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriversession_move_cursor(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriversession_press_key(uniffiSelf: bigint, input: Uint8Array): bigint;
uniffi_cua_driver_sdk_fn_method_cuadriversession_scroll(uniffiSelf: bigint, input: Uint8Array): bigint;
@@ -1645,12 +1711,15 @@ interface NativeModuleInterface {
uniffi_cua_driver_sdk_checksum_method_cuadriver_get_screen_size(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_get_session(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_get_session_state(): number;
+ uniffi_cua_driver_sdk_checksum_method_cuadriver_get_window_state(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_hotkey(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_invoke_menu(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_is_available(): number;
+ uniffi_cua_driver_sdk_checksum_method_cuadriver_list_apps(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_list_host_sessions_json(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_list_sessions(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_list_tools_json(): number;
+ uniffi_cua_driver_sdk_checksum_method_cuadriver_list_windows(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_metadata(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_move_cursor(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriver_press_key(): number;
@@ -1679,9 +1748,12 @@ interface NativeModuleInterface {
uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_screen_size(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_session(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_session_state(): number;
+ uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_window_state(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriversession_hotkey(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriversession_invoke_menu(): number;
+ uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_apps(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_sessions(): number;
+ uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_windows(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriversession_move_cursor(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriversession_press_key(): number;
uniffi_cua_driver_sdk_checksum_method_cuadriversession_scroll(): number;
diff --git a/libs/cua-driver/typescript/src/native/cua_driver_sdk.ts b/libs/cua-driver/typescript/src/native/cua_driver_sdk.ts
index db7cb9fb9a..0be9307d02 100644
--- a/libs/cua-driver/typescript/src/native/cua_driver_sdk.ts
+++ b/libs/cua-driver/typescript/src/native/cua_driver_sdk.ts
@@ -6,12 +6,12 @@
import nativeModule from "./cua_driver_sdk-ffi.js";
import { type UniffiRustFutureContinuationCallback, type UniffiForeignFutureDroppedCallback, type UniffiForeignFutureDroppedCallbackStruct, type UniffiVTableCallbackInterfaceCuaDriverSdkDriverActivityObserver, type UniffiForeignFutureResultRustBuffer, type UniffiForeignFutureCompleterustBuffer, type UniffiVTableCallbackInterfaceCuaDriverSdkDriverAuthorizationHost, type UniffiForeignFutureResultVoid, type UniffiForeignFutureCompletevoid, type UniffiVTableCallbackInterfaceCuaDriverSdkForeignDriverEnvelopeChannel,
} from "./cua_driver_sdk-ffi.js";
-import { type ActionResult, type ClickInput, type ClipboardReadInput, type ClipboardWriteInput, type DragInput, type EndSessionInput, type EndSessionOutput, type EscalateSessionInput, type GetAgentCursorStateInput, type GetCursorPositionInput, type GetDesktopStateInput, type GetScreenSizeInput, type GetSessionInput, type GetSessionStateInput, type HotkeyInput, type InvokeMenuInput, type ListSessionsInput, type ListSessionsOutput, type MoveCursorInput, type PressKeyInput, type ScrollInput, type SessionOutput, type SessionStateOutput, type SetAgentCursorEnabledInput, type SetAgentCursorMotionInput, type SetAgentCursorThemeInput, type SetWindowFrameInput, type StartSessionInput, type StartSessionOutput, type TypeTextInput, type VerifyStateInput, type VerifyStateOutput,
+import { type ActionResult, type ClickInput, type ClipboardReadInput, type ClipboardWriteInput, type DragInput, type EndSessionInput, type EndSessionOutput, type EscalateSessionInput, type GetAgentCursorStateInput, type GetCursorPositionInput, type GetDesktopStateInput, type GetScreenSizeInput, type GetSessionInput, type GetSessionStateInput, type GetWindowStateInput, type HotkeyInput, type InvokeMenuInput, type ListAppsInput, type ListAppsOutput, type ListSessionsInput, type ListSessionsOutput, type ListWindowsInput, type ListWindowsOutput, type MoveCursorInput, type PressKeyInput, type ScrollInput, type SessionOutput, type SessionStateOutput, type SetAgentCursorEnabledInput, type SetAgentCursorMotionInput, type SetAgentCursorThemeInput, type SetWindowFrameInput, type StartSessionInput, type StartSessionOutput, type TypeTextInput, type VerifyStateInput, type VerifyStateOutput, type WindowStateOutput,
} from "./cua_driver_contract.js";
import { type FfiConverter, type UniffiByteArray, type UniffiGcObject, type UniffiHandle, type UniffiObjectFactory, type UniffiReferenceHolder, type UniffiRustCallStatus, AbstractFfiConverterByteArray, FfiConverterArray, FfiConverterBool, FfiConverterInt32, FfiConverterObject, FfiConverterObjectWithCallbacks, FfiConverterOptional, FfiConverterUInt32, FfiConverterUInt64, FfiConverterUInt8, RustBuffer, UniffiAbstractObject, UniffiEnum, UniffiError, UniffiInternalError, UniffiResult, UniffiRustCaller, destructorGuardSymbol, pointerLiteralSymbol, uniffiCreateFfiConverterString, uniffiCreateRecord, uniffiRustCallAsync, uniffiTraitInterfaceCall, uniffiTraitInterfaceCallAsyncWithError, uniffiTraitInterfaceCallWithError, uniffiTypeNameSymbol, variantOrdinalSymbol,
} from "@ubjs/core";
import uniffiCuaDriverContractModule from "./cua_driver_contract.js";
-const { FfiConverterTypeActionResult, FfiConverterTypeClickInput, FfiConverterTypeClipboardReadInput, FfiConverterTypeClipboardWriteInput, FfiConverterTypeDragInput, FfiConverterTypeEndSessionInput, FfiConverterTypeEndSessionOutput, FfiConverterTypeEscalateSessionInput, FfiConverterTypeGetAgentCursorStateInput, FfiConverterTypeGetCursorPositionInput, FfiConverterTypeGetDesktopStateInput, FfiConverterTypeGetScreenSizeInput, FfiConverterTypeGetSessionInput, FfiConverterTypeGetSessionStateInput, FfiConverterTypeHotkeyInput, FfiConverterTypeInvokeMenuInput, FfiConverterTypeListSessionsInput, FfiConverterTypeListSessionsOutput, FfiConverterTypeMoveCursorInput, FfiConverterTypePressKeyInput, FfiConverterTypeScrollInput, FfiConverterTypeSessionOutput, FfiConverterTypeSessionStateOutput, FfiConverterTypeSetAgentCursorEnabledInput, FfiConverterTypeSetAgentCursorMotionInput, FfiConverterTypeSetAgentCursorThemeInput, FfiConverterTypeSetWindowFrameInput, FfiConverterTypeStartSessionInput, FfiConverterTypeStartSessionOutput, FfiConverterTypeTypeTextInput, FfiConverterTypeVerifyStateInput, FfiConverterTypeVerifyStateOutput } = uniffiCuaDriverContractModule.converters;
+const { FfiConverterTypeActionResult, FfiConverterTypeClickInput, FfiConverterTypeClipboardReadInput, FfiConverterTypeClipboardWriteInput, FfiConverterTypeDragInput, FfiConverterTypeEndSessionInput, FfiConverterTypeEndSessionOutput, FfiConverterTypeEscalateSessionInput, FfiConverterTypeGetAgentCursorStateInput, FfiConverterTypeGetCursorPositionInput, FfiConverterTypeGetDesktopStateInput, FfiConverterTypeGetScreenSizeInput, FfiConverterTypeGetSessionInput, FfiConverterTypeGetSessionStateInput, FfiConverterTypeGetWindowStateInput, FfiConverterTypeHotkeyInput, FfiConverterTypeInvokeMenuInput, FfiConverterTypeListAppsInput, FfiConverterTypeListAppsOutput, FfiConverterTypeListSessionsInput, FfiConverterTypeListSessionsOutput, FfiConverterTypeListWindowsInput, FfiConverterTypeListWindowsOutput, FfiConverterTypeMoveCursorInput, FfiConverterTypePressKeyInput, FfiConverterTypeScrollInput, FfiConverterTypeSessionOutput, FfiConverterTypeSessionStateOutput, FfiConverterTypeSetAgentCursorEnabledInput, FfiConverterTypeSetAgentCursorMotionInput, FfiConverterTypeSetAgentCursorThemeInput, FfiConverterTypeSetWindowFrameInput, FfiConverterTypeStartSessionInput, FfiConverterTypeStartSessionOutput, FfiConverterTypeTypeTextInput, FfiConverterTypeVerifyStateInput, FfiConverterTypeVerifyStateOutput, FfiConverterTypeWindowStateOutput } = uniffiCuaDriverContractModule.converters;
const uniffiCaller = new UniffiRustCaller(() => ({ code: 0 }));
const uniffiIsDebug =
@@ -3521,7 +3521,7 @@ export interface CuaDriverLike {
* remain downstream of the same public SDK runtime.
*/
callTool(name: string, argumentsJson: string, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
- click(input: ClickInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
+ click(input: ClickInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ActionResult>;
clipboardRead(input: ClipboardReadInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
clipboardWrite(input: ClipboardWriteInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
drag(input: DragInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
@@ -3534,9 +3534,11 @@ export interface CuaDriverLike {
getScreenSize(input: GetScreenSizeInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
getSession(input: GetSessionInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<SessionOutput>;
getSessionState(input: GetSessionStateInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<SessionStateOutput>;
+ getWindowState(input: GetWindowStateInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<WindowStateOutput>;
hotkey(input: HotkeyInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
invokeMenu(input: InvokeMenuInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
isAvailable(): boolean;
+ listApps(input: ListAppsInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ListAppsOutput>;
/**
* Content-free lifecycle summaries for the trusted host's runtime or
* host-lease namespace. This is deliberately separate from the
@@ -3548,6 +3550,7 @@ export interface CuaDriverLike {
* Canonical tool inventory for MCP and other protocol adapters.
*/
listToolsJson(asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<string>;
+ listWindows(input: ListWindowsInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ListWindowsOutput>;
metadata(asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<DriverMetadata>;
moveCursor(input: MoveCursorInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
pressKey(input: PressKeyInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
@@ -3872,7 +3875,7 @@ private constructor(pointer: UniffiHandle) {
}
}
- async click(input: ClickInput, asyncOpts_?: { signal: AbortSignal }): Promise<ToolResult> /*throws*/ {
+ async click(input: ClickInput, asyncOpts_?: { signal: AbortSignal }): Promise<ActionResult> /*throws*/ {
const __stack = uniffiIsDebug ? new Error().stack : undefined;
try {
return await uniffiRustCallAsync(
@@ -3891,7 +3894,7 @@ private constructor(pointer: UniffiHandle) {
// RustBuffer comes back via the shared `rust_future_complete_*`
// export. The bytes the runtime hands back must be deserialized
// here using the per-callable return-type converter.
- /*liftFunc:*/ FfiConverterTypeToolResult.lift.bind(FfiConverterTypeToolResult),
+ /*liftFunc:*/ FfiConverterTypeActionResult.lift.bind(FfiConverterTypeActionResult),
/*liftString:*/ FfiConverterString.lift.bind(FfiConverterString),
/*asyncOpts:*/ asyncOpts_,
/*errorHandler:*/ FfiConverterTypeDriverError.lift.bind(FfiConverterTypeDriverError)
@@ -4273,6 +4276,38 @@ private constructor(pointer: UniffiHandle) {
}
}
+ async getWindowState(input: GetWindowStateInput, asyncOpts_?: { signal: AbortSignal }): Promise<WindowStateOutput> /*throws*/ {
+ const __stack = uniffiIsDebug ? new Error().stack : undefined;
+ try {
+ return await uniffiRustCallAsync(
+ /*rustCaller:*/ uniffiCaller,
+ /*rustFutureFunc:*/ () => {
+ return nativeModule().uniffi_cua_driver_sdk_fn_method_cuadriver_get_window_state(
+ uniffiTypeCuaDriverObjectFactory.clonePointer(this),FfiConverterTypeGetWindowStateInput.lower(input, nativeModule().rustbuffer_alloc)
+ );
+ },
+ /*pollFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ /*cancelFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_cancel_rust_buffer,
+ /*completeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ /*freeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ // Async returns always go through the JS-side converter: the
+ // FFI symbol returns the future handle (u64), and the user-level
+ // RustBuffer comes back via the shared `rust_future_complete_*`
+ // export. The bytes the runtime hands back must be deserialized
+ // here using the per-callable return-type converter.
+ /*liftFunc:*/ FfiConverterTypeWindowStateOutput.lift.bind(FfiConverterTypeWindowStateOutput),
+ /*liftString:*/ FfiConverterString.lift.bind(FfiConverterString),
+ /*asyncOpts:*/ asyncOpts_,
+ /*errorHandler:*/ FfiConverterTypeDriverError.lift.bind(FfiConverterTypeDriverError)
+ );
+ } catch (__error: any) {
+ if (uniffiIsDebug && __error instanceof Error) {
+ __error.stack = __stack;
+ }
+ throw __error;
+ }
+ }
+
async hotkey(input: HotkeyInput, asyncOpts_?: { signal: AbortSignal }): Promise<ToolResult> /*throws*/ {
const __stack = uniffiIsDebug ? new Error().stack : undefined;
try {
@@ -4348,6 +4383,38 @@ private constructor(pointer: UniffiHandle) {
));
}
+ async listApps(input: ListAppsInput, asyncOpts_?: { signal: AbortSignal }): Promise<ListAppsOutput> /*throws*/ {
+ const __stack = uniffiIsDebug ? new Error().stack : undefined;
+ try {
+ return await uniffiRustCallAsync(
+ /*rustCaller:*/ uniffiCaller,
+ /*rustFutureFunc:*/ () => {
+ return nativeModule().uniffi_cua_driver_sdk_fn_method_cuadriver_list_apps(
+ uniffiTypeCuaDriverObjectFactory.clonePointer(this),FfiConverterTypeListAppsInput.lower(input, nativeModule().rustbuffer_alloc)
+ );
+ },
+ /*pollFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ /*cancelFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_cancel_rust_buffer,
+ /*completeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ /*freeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ // Async returns always go through the JS-side converter: the
+ // FFI symbol returns the future handle (u64), and the user-level
+ // RustBuffer comes back via the shared `rust_future_complete_*`
+ // export. The bytes the runtime hands back must be deserialized
+ // here using the per-callable return-type converter.
+ /*liftFunc:*/ FfiConverterTypeListAppsOutput.lift.bind(FfiConverterTypeListAppsOutput),
+ /*liftString:*/ FfiConverterString.lift.bind(FfiConverterString),
+ /*asyncOpts:*/ asyncOpts_,
+ /*errorHandler:*/ FfiConverterTypeDriverError.lift.bind(FfiConverterTypeDriverError)
+ );
+ } catch (__error: any) {
+ if (uniffiIsDebug && __error instanceof Error) {
+ __error.stack = __stack;
+ }
+ throw __error;
+ }
+ }
+
/**
* Content-free lifecycle summaries for the trusted host's runtime or
* host-lease namespace. This is deliberately separate from the
@@ -4452,6 +4519,38 @@ private constructor(pointer: UniffiHandle) {
}
}
+ async listWindows(input: ListWindowsInput, asyncOpts_?: { signal: AbortSignal }): Promise<ListWindowsOutput> /*throws*/ {
+ const __stack = uniffiIsDebug ? new Error().stack : undefined;
+ try {
+ return await uniffiRustCallAsync(
+ /*rustCaller:*/ uniffiCaller,
+ /*rustFutureFunc:*/ () => {
+ return nativeModule().uniffi_cua_driver_sdk_fn_method_cuadriver_list_windows(
+ uniffiTypeCuaDriverObjectFactory.clonePointer(this),FfiConverterTypeListWindowsInput.lower(input, nativeModule().rustbuffer_alloc)
+ );
+ },
+ /*pollFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ /*cancelFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_cancel_rust_buffer,
+ /*completeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ /*freeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ // Async returns always go through the JS-side converter: the
+ // FFI symbol returns the future handle (u64), and the user-level
+ // RustBuffer comes back via the shared `rust_future_complete_*`
+ // export. The bytes the runtime hands back must be deserialized
+ // here using the per-callable return-type converter.
+ /*liftFunc:*/ FfiConverterTypeListWindowsOutput.lift.bind(FfiConverterTypeListWindowsOutput),
+ /*liftString:*/ FfiConverterString.lift.bind(FfiConverterString),
+ /*asyncOpts:*/ asyncOpts_,
+ /*errorHandler:*/ FfiConverterTypeDriverError.lift.bind(FfiConverterTypeDriverError)
+ );
+ } catch (__error: any) {
+ if (uniffiIsDebug && __error instanceof Error) {
+ __error.stack = __stack;
+ }
+ throw __error;
+ }
+ }
+
async metadata(asyncOpts_?: { signal: AbortSignal }): Promise<DriverMetadata> /*throws*/ {
const __stack = uniffiIsDebug ? new Error().stack : undefined;
try {
@@ -4956,7 +5055,7 @@ const FfiConverterTypeCuaDriver = new FfiConverterObject(uniffiTypeCuaDriverObje
export interface CuaDriverSessionLike {
callTool(name: string, argumentsJson: string, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
- click(input: ClickInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
+ click(input: ClickInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ActionResult>;
clipboardRead(input: ClipboardReadInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
clipboardWrite(input: ClipboardWriteInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
/**
@@ -4975,9 +5074,12 @@ export interface CuaDriverSessionLike {
getScreenSize(input: GetScreenSizeInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
getSession(input: GetSessionInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<SessionOutput>;
getSessionState(input: GetSessionStateInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<SessionStateOutput>;
+ getWindowState(input: GetWindowStateInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<WindowStateOutput>;
hotkey(input: HotkeyInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
invokeMenu(input: InvokeMenuInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
+ listApps(input: ListAppsInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ListAppsOutput>;
listSessions(input: ListSessionsInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ListSessionsOutput>;
+ listWindows(input: ListWindowsInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ListWindowsOutput>;
moveCursor(input: MoveCursorInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
pressKey(input: PressKeyInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
scroll(input: ScrollInput, asyncOpts_?: { signal: AbortSignal }) /*throws*/: Promise<ToolResult>;
@@ -5042,7 +5144,7 @@ private constructor(pointer: UniffiHandle) {
}
}
- async click(input: ClickInput, asyncOpts_?: { signal: AbortSignal }): Promise<ToolResult> /*throws*/ {
+ async click(input: ClickInput, asyncOpts_?: { signal: AbortSignal }): Promise<ActionResult> /*throws*/ {
const __stack = uniffiIsDebug ? new Error().stack : undefined;
try {
return await uniffiRustCallAsync(
@@ -5061,7 +5163,7 @@ private constructor(pointer: UniffiHandle) {
// RustBuffer comes back via the shared `rust_future_complete_*`
// export. The bytes the runtime hands back must be deserialized
// here using the per-callable return-type converter.
- /*liftFunc:*/ FfiConverterTypeToolResult.lift.bind(FfiConverterTypeToolResult),
+ /*liftFunc:*/ FfiConverterTypeActionResult.lift.bind(FfiConverterTypeActionResult),
/*liftString:*/ FfiConverterString.lift.bind(FfiConverterString),
/*asyncOpts:*/ asyncOpts_,
/*errorHandler:*/ FfiConverterTypeDriverError.lift.bind(FfiConverterTypeDriverError)
@@ -5441,6 +5543,38 @@ private constructor(pointer: UniffiHandle) {
}
}
+ async getWindowState(input: GetWindowStateInput, asyncOpts_?: { signal: AbortSignal }): Promise<WindowStateOutput> /*throws*/ {
+ const __stack = uniffiIsDebug ? new Error().stack : undefined;
+ try {
+ return await uniffiRustCallAsync(
+ /*rustCaller:*/ uniffiCaller,
+ /*rustFutureFunc:*/ () => {
+ return nativeModule().uniffi_cua_driver_sdk_fn_method_cuadriversession_get_window_state(
+ uniffiTypeCuaDriverSessionObjectFactory.clonePointer(this),FfiConverterTypeGetWindowStateInput.lower(input, nativeModule().rustbuffer_alloc)
+ );
+ },
+ /*pollFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ /*cancelFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_cancel_rust_buffer,
+ /*completeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ /*freeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ // Async returns always go through the JS-side converter: the
+ // FFI symbol returns the future handle (u64), and the user-level
+ // RustBuffer comes back via the shared `rust_future_complete_*`
+ // export. The bytes the runtime hands back must be deserialized
+ // here using the per-callable return-type converter.
+ /*liftFunc:*/ FfiConverterTypeWindowStateOutput.lift.bind(FfiConverterTypeWindowStateOutput),
+ /*liftString:*/ FfiConverterString.lift.bind(FfiConverterString),
+ /*asyncOpts:*/ asyncOpts_,
+ /*errorHandler:*/ FfiConverterTypeDriverError.lift.bind(FfiConverterTypeDriverError)
+ );
+ } catch (__error: any) {
+ if (uniffiIsDebug && __error instanceof Error) {
+ __error.stack = __stack;
+ }
+ throw __error;
+ }
+ }
+
async hotkey(input: HotkeyInput, asyncOpts_?: { signal: AbortSignal }): Promise<ToolResult> /*throws*/ {
const __stack = uniffiIsDebug ? new Error().stack : undefined;
try {
@@ -5505,6 +5639,38 @@ private constructor(pointer: UniffiHandle) {
}
}
+ async listApps(input: ListAppsInput, asyncOpts_?: { signal: AbortSignal }): Promise<ListAppsOutput> /*throws*/ {
+ const __stack = uniffiIsDebug ? new Error().stack : undefined;
+ try {
+ return await uniffiRustCallAsync(
+ /*rustCaller:*/ uniffiCaller,
+ /*rustFutureFunc:*/ () => {
+ return nativeModule().uniffi_cua_driver_sdk_fn_method_cuadriversession_list_apps(
+ uniffiTypeCuaDriverSessionObjectFactory.clonePointer(this),FfiConverterTypeListAppsInput.lower(input, nativeModule().rustbuffer_alloc)
+ );
+ },
+ /*pollFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ /*cancelFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_cancel_rust_buffer,
+ /*completeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ /*freeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ // Async returns always go through the JS-side converter: the
+ // FFI symbol returns the future handle (u64), and the user-level
+ // RustBuffer comes back via the shared `rust_future_complete_*`
+ // export. The bytes the runtime hands back must be deserialized
+ // here using the per-callable return-type converter.
+ /*liftFunc:*/ FfiConverterTypeListAppsOutput.lift.bind(FfiConverterTypeListAppsOutput),
+ /*liftString:*/ FfiConverterString.lift.bind(FfiConverterString),
+ /*asyncOpts:*/ asyncOpts_,
+ /*errorHandler:*/ FfiConverterTypeDriverError.lift.bind(FfiConverterTypeDriverError)
+ );
+ } catch (__error: any) {
+ if (uniffiIsDebug && __error instanceof Error) {
+ __error.stack = __stack;
+ }
+ throw __error;
+ }
+ }
+
async listSessions(input: ListSessionsInput, asyncOpts_?: { signal: AbortSignal }): Promise<ListSessionsOutput> /*throws*/ {
const __stack = uniffiIsDebug ? new Error().stack : undefined;
try {
@@ -5537,6 +5703,38 @@ private constructor(pointer: UniffiHandle) {
}
}
+ async listWindows(input: ListWindowsInput, asyncOpts_?: { signal: AbortSignal }): Promise<ListWindowsOutput> /*throws*/ {
+ const __stack = uniffiIsDebug ? new Error().stack : undefined;
+ try {
+ return await uniffiRustCallAsync(
+ /*rustCaller:*/ uniffiCaller,
+ /*rustFutureFunc:*/ () => {
+ return nativeModule().uniffi_cua_driver_sdk_fn_method_cuadriversession_list_windows(
+ uniffiTypeCuaDriverSessionObjectFactory.clonePointer(this),FfiConverterTypeListWindowsInput.lower(input, nativeModule().rustbuffer_alloc)
+ );
+ },
+ /*pollFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_poll_rust_buffer,
+ /*cancelFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_cancel_rust_buffer,
+ /*completeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_complete_rust_buffer,
+ /*freeFunc:*/ nativeModule().ffi_cua_driver_sdk_rust_future_free_rust_buffer,
+ // Async returns always go through the JS-side converter: the
+ // FFI symbol returns the future handle (u64), and the user-level
+ // RustBuffer comes back via the shared `rust_future_complete_*`
+ // export. The bytes the runtime hands back must be deserialized
+ // here using the per-callable return-type converter.
+ /*liftFunc:*/ FfiConverterTypeListWindowsOutput.lift.bind(FfiConverterTypeListWindowsOutput),
+ /*liftString:*/ FfiConverterString.lift.bind(FfiConverterString),
+ /*asyncOpts:*/ asyncOpts_,
+ /*errorHandler:*/ FfiConverterTypeDriverError.lift.bind(FfiConverterTypeDriverError)
+ );
+ } catch (__error: any) {
+ if (uniffiIsDebug && __error instanceof Error) {
+ __error.stack = __stack;
+ }
+ throw __error;
+ }
+ }
+
async moveCursor(input: MoveCursorInput, asyncOpts_?: { signal: AbortSignal }): Promise<ToolResult> /*throws*/ {
const __stack = uniffiIsDebug ? new Error().stack : undefined;
try {
@@ -6726,7 +6924,7 @@ function uniffiEnsureInitialized() {
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_call_tool() !== 24493) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_call_tool");
}
- if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_click() !== 56077) {
+ if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_click() !== 22807) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_click");
}
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_clipboard_read() !== 58238) {
@@ -6765,6 +6963,9 @@ function uniffiEnsureInitialized() {
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_get_session_state() !== 49966) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_get_session_state");
}
+ if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_get_window_state() !== 62142) {
+ throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_get_window_state");
+ }
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_hotkey() !== 53333) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_hotkey");
}
@@ -6774,6 +6975,9 @@ function uniffiEnsureInitialized() {
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_is_available() !== 42961) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_is_available");
}
+ if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_list_apps() !== 35505) {
+ throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_list_apps");
+ }
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_list_host_sessions_json() !== 13193) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_list_host_sessions_json");
}
@@ -6783,6 +6987,9 @@ function uniffiEnsureInitialized() {
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_list_tools_json() !== 33039) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_list_tools_json");
}
+ if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_list_windows() !== 52601) {
+ throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_list_windows");
+ }
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriver_metadata() !== 55026) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriver_metadata");
}
@@ -6828,7 +7035,7 @@ function uniffiEnsureInitialized() {
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_call_tool() !== 52859) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_call_tool");
}
- if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_click() !== 22303) {
+ if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_click() !== 50816) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_click");
}
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_clipboard_read() !== 23560) {
@@ -6867,15 +7074,24 @@ function uniffiEnsureInitialized() {
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_session_state() !== 55370) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_session_state");
}
+ if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_window_state() !== 21266) {
+ throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_get_window_state");
+ }
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_hotkey() !== 12820) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_hotkey");
}
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_invoke_menu() !== 2742) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_invoke_menu");
}
+ if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_apps() !== 51520) {
+ throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_apps");
+ }
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_sessions() !== 32840) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_sessions");
}
+ if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_windows() !== 20571) {
+ throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_list_windows");
+ }
if (nativeModule().uniffi_cua_driver_sdk_checksum_method_cuadriversession_move_cursor() !== 35768) {
throw new UniffiInternalError.ApiChecksumMismatch("uniffi_cua_driver_sdk_checksum_method_cuadriversession_move_cursor");
}
diff --git a/libs/cua-driver/typescript/test/compatibility-contract.test.mjs b/libs/cua-driver/typescript/test/compatibility-contract.test.mjs
index 4c64358086..0beee1cf7f 100644
--- a/libs/cua-driver/typescript/test/compatibility-contract.test.mjs
+++ b/libs/cua-driver/typescript/test/compatibility-contract.test.mjs
@@ -14,6 +14,20 @@ const fixture = JSON.parse(
const normalize = (value) => value.replace(/\s+/g, " ").trim()
+test("current native-window methods return typed outputs", () => {
+ const declarations = normalize(fs.readFileSync(
+ path.join(packageRoot, "dist", "native", "cua_driver_sdk.d.ts"), "utf8",
+ ))
+ for (const [method, input, output] of [
+ ["listApps", "ListAppsInput", "ListAppsOutput"],
+ ["listWindows", "ListWindowsInput", "ListWindowsOutput"],
+ ["getWindowState", "GetWindowStateInput", "WindowStateOutput"],
+ ["click", "ClickInput", "ActionResult"],
+ ]) {
+ assert.ok(declarations.includes(`${method}(input: ${input}, asyncOpts_?: { signal: AbortSignal; }): Promise<${output}>;`), method)
+ }
+})
+
test("released package exports and declarations remain available", () => {
const packageJson = JSON.parse(
fs.readFileSync(path.join(packageRoot, "package.json"), "utf8"),
diff --git a/libs/cua-driver/typescript/test/electron-main-fixture.mjs b/libs/cua-driver/typescript/test/electron-main-fixture.mjs
index 76a85e2575..787ec22f36 100644
--- a/libs/cua-driver/typescript/test/electron-main-fixture.mjs
+++ b/libs/cua-driver/typescript/test/electron-main-fixture.mjs
@@ -27,7 +27,7 @@ const server = net.createServer(socket => {
const request = JSON.parse(buffer.split("\\n", 1)[0]);
const result = request.method === "metadata" ? {
driver_version: ${JSON.stringify(packageVersion)},
- contract_version: hostBundleId.endsWith(".failure") ? "incompatible" : "0.7.0",
+ contract_version: hostBundleId.endsWith(".failure") ? "incompatible" : "0.8.0",
tools_list_schema_version: "1",
capability_version: "1",
mcp_protocol_version: "2025-06-18",
diff --git a/libs/cua-driver/typescript/test/native-daemon-fixture.mjs b/libs/cua-driver/typescript/test/native-daemon-fixture.mjs
index b127599ba7..f31343e355 100644
--- a/libs/cua-driver/typescript/test/native-daemon-fixture.mjs
+++ b/libs/cua-driver/typescript/test/native-daemon-fixture.mjs
@@ -18,7 +18,7 @@ const server = net.createServer((connection) => {
ok: true,
result: {
driver_version: "0.12.6",
- contract_version: "0.7.0",
+ contract_version: "0.8.0",
tools_list_schema_version: "1",
capability_version: "1",
mcp_protocol_version: "2025-06-18",
@@ -31,7 +31,7 @@ const server = net.createServer((connection) => {
return
}
process.send?.({ request })
- const structuredContent =
+ let structuredContent =
request.name === "verify_state"
? {
status: "satisfied",
@@ -45,6 +45,14 @@ const server = net.createServer((connection) => {
route: "global_input",
delivery: { mode: "not_applicable" },
}
+ let isError = false
+ if (request.name === "list_apps") structuredContent = { apps: [{ pid: 42, name: "Editor", running: true, active: false }] }
+ if (request.name === "list_windows") structuredContent = { windows: [{ pid: 42, window_id: 123, app_name: "Editor", title: "Document", bounds: { x: 0, y: 0, width: 800, height: 600 }, is_on_screen: true, z_index: null }] }
+ if (request.name === "get_window_state") structuredContent = { pid: 42, window_id: 123, snapshot_id: "snapshot-1", screenshot_width: 800, screenshot_height: 600, elements: [{ element_index: 0, role: "button", depth: 0, element_token: "fresh-token", label: "Save" }] }
+ if (request.name === "click" && (request.args.element_token === "stale-token" || request.args.target?.window_id === 124)) {
+ isError = true
+ structuredContent = { code: request.args.element_token === "stale-token" ? "stale_element_token" : "element_target_mismatch" }
+ }
connection.end(
`${JSON.stringify({
ok: true,
@@ -54,12 +62,12 @@ const server = net.createServer((connection) => {
{ type: "image", mimeType: "image/png", data: "cG5n" },
],
structuredContent,
- isError: false,
+ isError,
},
})}\n`,
)
completedCalls += 1
- if (completedCalls === 2) server.close()
+ if (completedCalls === 8) server.close()
})
})
diff --git a/libs/cua-driver/typescript/test/native-loader.test.mjs b/libs/cua-driver/typescript/test/native-loader.test.mjs
index 5557fd316b..710a67baa4 100644
--- a/libs/cua-driver/typescript/test/native-loader.test.mjs
+++ b/libs/cua-driver/typescript/test/native-loader.test.mjs
@@ -53,7 +53,7 @@ const server = net.createServer(socket => {
const request = JSON.parse(buffer.split("\\n", 1)[0]);
const result = request.method === "metadata" ? {
driver_version: "0.10.0",
- contract_version: "0.7.0",
+ contract_version: "0.8.0",
tools_list_schema_version: "1",
capability_version: "1",
mcp_protocol_version: "2025-06-18",
@@ -114,7 +114,7 @@ test(
fixture.on("error", reject)
fixture.on("message", (message) => {
if (message.request) requests.push(message.request)
- if (requests.length === 2) resolve(requests)
+ if (requests.length === 8) resolve(requests)
})
}),
)
@@ -126,10 +126,12 @@ test(
const {
ActionEffect,
ActionRoute,
+ ActionTarget,
+ ClickPosition,
+ InputDeliveryMode,
ClickButton,
ClickInput,
CuaDriver,
- DesktopScope,
StatePredicate,
VerificationStatus,
VerifyStateInput,
@@ -153,6 +155,9 @@ test(
"getSessionState",
"endSession",
"getDesktopState",
+ "listApps",
+ "listWindows",
+ "getWindowState",
"getScreenSize",
"getCursorPosition",
"moveCursor",
@@ -185,14 +190,47 @@ test(
)
const actionResult = await driver.click(
ClickInput.new({
- x: 12,
- y: 34,
- scope: DesktopScope.Desktop,
+ position: new ClickPosition.Coordinates({ x: 12, y: 34 }),
+ target: new ActionTarget.Desktop({ displayId: "primary" }),
+ deliveryMode: InputDeliveryMode.Foreground,
session: "node-run",
button: ClickButton.Left,
count: 1,
}),
)
+ const apps = await driver.listApps(sdk.ListAppsInput.new({}))
+ assert.equal(apps.apps[0].name, "Editor")
+ const windows = await driver.listWindows(sdk.ListWindowsInput.new({ pid: 42, onScreenOnly: true }))
+ assert.equal(windows.windows[0].windowId, 123n)
+ assert.equal(windows.windows[0].zIndex, undefined)
+ const state = await driver.getWindowState(sdk.GetWindowStateInput.new({
+ pid: 42, windowId: 123n, session: "node-run", query: "Save",
+ includeScreenshot: true, includeAccessibilityTree: true,
+ maxElements: 10, maxDepth: 3, maxDimension: 800,
+ }))
+ assert.equal(state.snapshotId, "snapshot-1")
+ assert.equal(state.elements[0].label, "Save")
+ assert.equal(state.images[0].mimeType, "image/png")
+ assert.equal(state.images[0].dataBase64, "cG5n")
+ const tokenClick = (token, windowId) => ClickInput.new({
+ target: new ActionTarget.Window({ pid: 42, windowId }),
+ position: new ClickPosition.Element({ elementToken: token }),
+ deliveryMode: InputDeliveryMode.Background, session: "node-run",
+ })
+ const freshAction = await driver.click(tokenClick(state.elements[0].elementToken, 123n))
+ assert.equal(freshAction.effect, ActionEffect.Unverifiable)
+ // Service fixtures qualify SDK error propagation, not native token validation.
+ for (const [token, windowId, code] of [
+ ["stale-token", 123n, "stale_element_token"],
+ ["fresh-token", 124n, "element_target_mismatch"],
+ ]) {
+ await assert.rejects(driver.click(tokenClick(token, windowId)), error => {
+ assert.equal(sdk.DriverError.Tool.instanceOf(error), true)
+ assert.equal(error.inner.tool, "click")
+ assert.equal(error.inner.errorCode, code)
+ return true
+ })
+ }
await requestsPromise
driver.uniffiDestroy()
@@ -201,9 +239,22 @@ test(
assert.equal(verificationResult.action, undefined)
assert.equal(verificationResult.verification.status, VerificationStatus.Satisfied)
assert.equal(actionResult.verification, undefined)
- assert.equal(actionResult.action.effect, ActionEffect.Unverifiable)
- assert.equal(actionResult.action.route, ActionRoute.GlobalInput)
+ assert.equal(actionResult.effect, ActionEffect.Unverifiable)
+ assert.equal(actionResult.route, ActionRoute.GlobalInput)
assert.equal("verified" in actionResult, false)
+ assert.deepEqual(requests[2].args, {})
+ assert.deepEqual(requests[3].args, { pid: 42, on_screen_only: true })
+ assert.deepEqual(requests[4].args, {
+ pid: 42, window_id: 123, session: "node-run", query: "Save",
+ include_screenshot: true, include_accessibility_tree: true,
+ max_elements: 10, max_depth: 3, max_dimension: 800,
+ })
+ assert.deepEqual(requests[5].args, {
+ target: { kind: "window", pid: 42, window_id: 123 },
+ element_token: "fresh-token", delivery_mode: "background", session: "node-run",
+ })
+ assert.equal(requests[6].args.element_token, "stale-token")
+ assert.equal(requests[7].args.target.window_id, 124)
assert.equal(requests[0].name, "verify_state")
assert.deepEqual(requests[0].args, {
pid: 123,
@@ -219,7 +270,8 @@ test(
assert.deepEqual(requests[1].args, {
x: 12,
y: 34,
- scope: "desktop",
+ target: { kind: "desktop", display_id: "primary" },
+ delivery_mode: "foreground",
session: "node-run",
button: "left",
count: 1,
diff --git a/libs/python/cua-sandbox/tests/test_typed_driver_native.py b/libs/python/cua-sandbox/tests/test_typed_driver_native.py
index b666b76f6e..9753ddef38 100644
--- a/libs/python/cua-sandbox/tests/test_typed_driver_native.py
+++ b/libs/python/cua-sandbox/tests/test_typed_driver_native.py
@@ -106,23 +106,26 @@ async def test_real_canonical_action_target_serializes_over_fleet(
await sandbox._connect()
try:
async with sandbox.driver.connect() as driver:
- await driver.click(
- sdk.ClickInput(
- x=12.5,
- y=34.5,
- target=getattr(sdk.ActionTarget, variant)(**fields),
- scope=None,
- session=sandbox.driver.session_name(driver),
- button=None,
- count=None,
+ with pytest.raises(sdk.DriverError.Tool) as error:
+ await driver.click(
+ sdk.ClickInput(
+ position=sdk.ClickPosition.COORDINATES(x=12.5, y=34.5),
+ target=getattr(sdk.ActionTarget, variant)(**fields),
+ delivery_mode=sdk.InputDeliveryMode.FOREGROUND,
+ session=sandbox.driver.session_name(driver),
+ button=None,
+ count=None,
+ )
)
- )
+ assert error.value.tool == "click"
+ assert error.value.error_code == "foreground_required"
request = transport.events[-1][3]
assert request["name"] == "click"
assert request["arguments"] == {
"x": 12.5,
"y": 34.5,
"target": expected,
+ "delivery_mode": "foreground",
"session": "session-1",
}
finally:
diff --git a/rfcs/3682-typed-native-window-sdk.md b/rfcs/3682-typed-native-window-sdk.md
new file mode 100644
index 0000000000..0c6d1db3fc
--- /dev/null
+++ b/rfcs/3682-typed-native-window-sdk.md
@@ -0,0 +1,100 @@
+---
+title: Typed native-window SDK flow and explicit click addressing
+authors:
+ - f-trycua
+created: 2026-09-09
+last_updated: 2026-09-09
+status: accepted
+discussion: https://github.com/trycua/cua/issues/3682
+rfc_pr: https://github.com/trycua/cua/pull/3683
+implementation:
+ - https://github.com/trycua/cua/pull/3683
+supersedes:
+superseded_by:
+---
+
+# RFC: Typed native-window SDK flow
+
+## Decision
+
+Expose the existing native discover / snapshot / act / verify flow through
+generated Python and TypeScript methods. Accept an intentional breaking SDK
+revision so one click operation supports either coordinates or a snapshot-bound
+element token, with an explicit target and delivery mode.
+
+The maintainer selected this scope for implementation. The shortened proposal
+window allows the SDK to converge on existing native capabilities in one
+revision; cross-platform verification and implementation review remain gates.
+This changes the additive SDK-surface assumption for this bounded flow in
+[RFC 2549](2549-cua-driver-sdk-owned-runtime.md), without changing its runtime
+topology. It does not change the snapshot lifetime work in issue #3473 or add
+the post-action rediscovery proposed in pull request #3373.
+
+## Motivation
+
+In SDK 0.25.0, consumers must call `callTool("get_window_state", ...)` and parse
+`structuredJson` to inspect a native window. The typed `click` requires x/y and
+cannot express an element token or background delivery. The native runtime
+already supports these operations. The SDK should expose the full flow without
+JSON construction or parsing in application code.
+
+## Public contract
+
+- Both drivers and bound sessions expose `list_apps`, `list_windows`, and
+ `get_window_state` (camelCase in TypeScript).
+- Discovery and snapshot methods return typed successful results. Snapshots
+ include exact window identity, elements and opaque tokens, bounds, screenshot
+ data/metadata, and completeness/degradation information. Missing platform
+ information remains optional; it is never invented.
+- `click` accepts an explicit coordinate-or-element-token sum type, an exact
+ window or desktop target, and an explicit background/foreground mode. It
+ returns the existing typed action result. Refusals become SDK errors carrying
+ the native error code and message.
+- Shared Rust declarations generate both bindings. They serialize to the
+ existing runtime tool inputs; CLI/MCP wire behavior stays compatible.
+- Generic `call_tool` / `callTool` remains available for platform extensions.
+
+## Platform and trust boundaries
+
+Native adapters retain discovery, capture and delivery. Existing permissions,
+session authority, snapshot replacement and token ownership remain enforced.
+Background delivery never silently escalates to foreground. A dispatched action
+does not establish application effect: callers take a fresh snapshot or use
+`verify_state`. OS/compositor limitations remain explicit refusals or missing
+optional information. Snapshot data is not added to telemetry.
+
+## Alternatives
+
+An additive `clickElement` preserves the old call but splits one operation into
+competing APIs. Independent optional x/y/token fields admit ambiguous input.
+Generic JSON calls leave output typing and safe addressing to each application.
+Use one generated sum type instead.
+
+## Compatibility and migration
+
+Advance the contract shape version. Existing SDK coordinate calls migrate to
+the addressing type, explicit target and delivery mode, and direct action
+result. Update compatibility fixtures to record the intentional change while
+retaining checks for the rest of the public surface. Regenerate bindings and
+publish Python and TypeScript migration examples. Pinning the previous SDK or
+reverting this product change provides a rollback path.
+
+## Implementation and acceptance
+
+1. Define portable Rust discovery/snapshot records and click addressing; prove
+ serialization against every native backend schema.
+2. Expose typed driver/session methods and normalize successful results and
+ refusals once in the shared SDK boundary.
+3. Regenerate bindings, migrate fixtures/examples, and compile and execute the
+ complete flow through Python and TypeScript.
+4. Run focused contract/SDK tests during development. Certify the stable SHA
+ with canonical Windows/Linux desktop E2E and the macOS Lume matrix, including
+ application-effect, token freshness, focus, cursor and input-isolation checks.
+5. Complete CI and independent review before landing. Record exact candidate
+ evidence in the implementation pull request.
+
+## Remaining implementation questions
+
+Finalize portable optional snapshot fields and the binding-friendly addressing
+representation from the current platform schemas. The implementation must not
+change native delivery semantics to make the SDK appear more capable.