Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
feat(desktop): add opt-in raw macOS click route (#2414)
Max61e315abef58
diff --git a/.gitattributes b/.gitattributesindex 8cc1428c9..f30ee3976 100644--- a/.gitattributes+++ b/.gitattributes@@ -1,3 +1,5 @@# Byte-exact generated resource and released-template migration fixture.vibe/show_router.tsx text eol=lftests/fixtures/show_pages/router-history-pre-ssr.tsx text eol=lf+desktop/cua-driver/patches/*.patch text eol=lf+desktop/cua-driver/tools-v0.31.0.json text eol=lfdiff --git a/.github/workflows/desktop-package.yml b/.github/workflows/desktop-package.ymlindex d3893d049..587273e88 100644--- a/.github/workflows/desktop-package.yml+++ b/.github/workflows/desktop-package.yml@@ -209,6 +209,14 @@ jobs:echo "apple_signing=absent" >> "$GITHUB_ENV"fi+ - name: Verify prepared Cua Driver provenance+ if: runner.os == 'macOS'+ run: >+ python scripts/desktop_release.py verify-driver-prepared+ --target "${{ matrix.target }}"+ --driver "desktop/src-tauri/binaries/cua-driver-${{ matrix.target }}"+ --provenance-path "desktop/src-tauri/binaries/cua-driver-${{ matrix.target }}.provenance.json"+- name: Build self-contained installerworking-directory: desktoprun: >@@ -321,6 +329,7 @@ jobs:if [ "$RUNNER_OS" = "macOS" ]; thensigning="${signature_state}"$'\ndmg-unsigned-unnotarized'driver_args=(+ --prepared-driver "desktop/src-tauri/binaries/cua-driver-${{ matrix.target }}"--driver "$driver_packaged_path"--driver-provenance-path "$driver_provenance_path"--driver-cdhash "$driver_packaged_cdhash"diff --git a/core/computer_server.py b/core/computer_server.pyindex ce8f01111..5e4922b85 100644--- a/core/computer_server.py+++ b/core/computer_server.py@@ -670,8 +670,52 @@ def _focus_shortcut(arguments: Mapping[str, Any], tool_name: str) -> bool:def _validate_input(tool_name: str, arguments: Mapping[str, Any]) -> None:+ if "click_mode" in arguments and tool_name != "click":+ raise ComputerServerError(+ "invalid_arguments",+ "click_mode is valid only for the click tool.",+ )if tool_name not in _INPUT_TOOLS:return+ if "click_mode" in arguments:+ click_mode = arguments.get("click_mode")+ if not isinstance(click_mode, str) or click_mode not in {"auto", "raw"}:+ raise ComputerServerError(+ "invalid_arguments",+ 'click_mode must be "auto" or "raw".',+ )+ if click_mode == "raw":+ pid = arguments.get("pid")+ window_id = arguments.get("window_id")+ x = arguments.get("x")+ y = arguments.get("y")+ count = arguments.get("count", 1)+ modifiers = arguments.get("modifier", [])+ if (+ isinstance(pid, bool)+ or not isinstance(pid, int)+ or isinstance(window_id, bool)+ or not isinstance(window_id, int)+ or isinstance(x, bool)+ or not isinstance(x, (int, float))+ or isinstance(y, bool)+ or not isinstance(y, (int, float))+ or arguments.get("element_token") is not None+ or arguments.get("target") is not None+ or arguments.get("action", "press") not in {"press", "click"}+ or arguments.get("button", "left") != "left"+ or arguments.get("delivery_mode", "background") != "background"+ or isinstance(count, bool)+ or not isinstance(count, int)+ or count != 1+ or not isinstance(modifiers, list)+ or modifiers+ ):+ raise ComputerServerError(+ "invalid_arguments",+ 'click_mode="raw" requires an exact pid + window_id '+ "background left-click with x/y, count=1, and no modifiers.",+ )if arguments.get("delivery_mode") == "foreground":raise ComputerServerError("foreground_forbidden",diff --git a/core/computer_use.py b/core/computer_use.pyindex 44211f4a9..f949ef2a3 100644--- a/core/computer_use.py+++ b/core/computer_use.py@@ -36,7 +36,7 @@ COMPUTER_USE_LEASE_FILE = "computer-lease.json"COMPUTER_USE_LEASE_LOCK_FILE = "computer-lease.lock"COMPUTER_USE_RECONCILE_INTERVAL_SECONDS = 2.0COMPUTER_USE_TOOL_SNAPSHOT_SHA256 = (- "b03c3e48d1b00c8fe7c0e8b9813eb5ea104ad38313672fc4b68af203a827f43b"+ "4257e598f5a6ad7e632b48e96c03257d7c5ed618a93b72820b2d4fef0f196acf")COMPUTER_USE_TOOL_NAMES = (diff --git a/desktop/README.md b/desktop/README.mdindex c1f790873..548447a82 100644--- a/desktop/README.md+++ b/desktop/README.md@@ -170,10 +170,10 @@ DMG/EXE download has a target-qualified `.SHA256SUMS`, `.SIGNATURE`,the verified layer; it is not a cryptographic signature. Hashes and sourcemetadata are checked before upload and downloaded desktop bytes are checkedagain before publication. New `.SOURCE.json` records use schema 2; every macOS-record includes the pinned Cua Driver source, archive, extracted, thinned,-packaged, and code-signing provenance. The verifier still accepts already-published schema-1 records as legacy input. A failed build or check leaves the-release unpublished.+record includes the pinned Cua Driver source archive, patch, tool snapshot,+prepared binary, packaged binary, and code-signing provenance. The verifier+still accepts already published schema-1 records as legacy input. A failed+build or check leaves the release unpublished.Existing asset bytes cannot be overwritten; a changed build requires a new rc.Already-published releases cannot be retrofitted with missing desktop assets.diff --git a/desktop/cua-driver/PATCHES.md b/desktop/cua-driver/PATCHES.mdnew file mode 100644index 000000000..84f941071--- /dev/null+++ b/desktop/cua-driver/PATCHES.md@@ -0,0 +1,53 @@+# Pinned Cua Driver Patches++Avibe consumes the MIT-licensed `cua-driver-rs-v0.31.0` source at commit+`5272e492d61b96caf08e3bf434d91126c1f3dccc`. The release binary is not used+for this variant. The package workflow downloads the pinned source archive,+verifies its SHA-256, applies the patch recorded in `sources.json`, builds the+requested macOS target, and records the resulting provenance.++## `avibe-raw-single-click-v1`++Patch: `patches/0001-macos-raw-single-click.patch`++This patch adds the opt-in `click.click_mode` selector:++- omitted or `"auto"` preserves the existing AX-first `count=1` behavior;+- `"raw"` is accepted only for an exact `pid` and `window_id`, window pixel+ coordinates, background left-click delivery, `action` `press` or `click`,+ `count=1`, no modifiers, and no element or portable target;+- `"raw"` bypasses the macOS pixel-path AX hit-test/`AXPress` optimization and+ uses the existing routed CGEvent path;+- unknown values and invalid combinations return structured+ `invalid_arguments`.++The patch also adds the test-owned AppKit hollow-AX target and its ignored+background acceptance harness. The target reports AX presses separately from+native `mouseDown`/`mouseUp` delivery, so the acceptance can prove one raw+mouse pair and unchanged default behavior without relying on the tool result+alone.++The upstream investigation found no consumable maintainer-accepted contract+with these semantics. The relevant discussions are:++- [PR #3922](https://github.com/trycua/cua/pull/3922)+- [PR #3539](https://github.com/trycua/cua/pull/3539)+- [PR #4632](https://github.com/trycua/cua/pull/4632)+- [PR #4208](https://github.com/trycua/cua/pull/4208)+- [Issue #4350](https://github.com/trycua/cua/issues/4350)+- [Issue #2874](https://github.com/trycua/cua/issues/2874)++The exact patch SHA-256, source archive SHA-256, tool snapshot SHA-256,+contract, and upgrade plan are duplicated in `sources.json`. The release+provenance uses schema 2 and includes those hashes plus the prepared binary+hash and post-signing packaged hash.++When upstream exposes an opt-in raw `count=1` macOS click selector with the+same default-compatible contract, remove this patch and pin the upstream+release. Otherwise, rebase the patch onto the next pinned source commit,+rerun the fixture acceptance, refresh the source and snapshot hashes, and+review the Avibe forwarding admission again.++The bounded foreground mode remains deferred. It requires separate evidence+for apps that reject both AX actions and background synthetic events; it is+not an implicit fallback for this patch.diff --git a/desktop/cua-driver/patches/0001-macos-raw-single-click.patch b/desktop/cua-driver/patches/0001-macos-raw-single-click.patchnew file mode 100644index 000000000..a0d636d25--- /dev/null+++ b/desktop/cua-driver/patches/0001-macos-raw-single-click.patch@@ -0,0 +1,406 @@+diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+index 4fa4e14ab..ec42f9f9b 100644+--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs++++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+@@ -81,7 +81,13 @@ impl Harness {+ pointer_oracle: Option<&Path>,+ keep_ordered_front: bool,+ ) -> Self {+- let harness = Self::spawn(command_oracle, pointer_oracle, keep_ordered_front, None);++ let harness = Self::spawn(++ command_oracle,++ pointer_oracle,++ keep_ordered_front,++ None,++ false,++ );+ // Settle for window creation + activation.+ std::thread::sleep(Duration::from_millis(800));+ harness.await_presented();+@@ -91,5 +97,12 @@ impl Harness {+ /// Launch a harness that holds off entering its run loop for `delay`+ /// after registering its window, without waiting for its launch posture.+ fn launch_slowly(delay: Duration) -> Self {+- Self::spawn(None, None, false, Some(delay))++ Self::spawn(None, None, false, Some(delay), false)++ }++++ fn launch_with_raw_click_target() -> Self {++ let harness = Self::spawn(None, None, false, None, true);++ std::thread::sleep(Duration::from_millis(800));++ harness.await_presented();++ harness+ }+@@ -99,6 +112,7 @@ impl Harness {+ pointer_oracle: Option<&Path>,+ keep_ordered_front: bool,+ launch_delay: Option<Duration>,++ raw_click_target: bool,+ ) -> Self {+ let exe = harness_exe();+ assert!(+@@ -122,6 +136,9 @@ impl Harness {+ if let Some(delay) = launch_delay {+ command.env("CUA_APPKIT_LAUNCH_DELAY_MS", delay.as_millis().to_string());+ }++ if raw_click_target {++ command.env("CUA_APPKIT_RAW_CLICK_TARGET", "1");++ }+ let app = command+ .spawn()+ .unwrap_or_else(|error| panic!("launch AppKit harness {exe:?}: {error}"));+@@ -1649,3 +1666,85 @@ fn harness_appkit_counter_px_background() {++#[test]++#[ignore]++fn harness_appkit_raw_single_click_bypasses_hollow_ax_press() {++ let case = native_background_case(++ "appkit",++ "raw_single_click",++ Targeting::Px,++ DriverRoute::MacosCgEventPid,++ );++ execute_case(case, |evidence| {++ let mut driver = McpDriver::spawn_macos_daemon_proxy_named("appkit-raw-single-click")++ .expect("start macOS daemon proxy");++ *evidence = recording_evidence(driver.recording_dir());++ let harness = Harness::launch_with_raw_click_target();++ let (wid, _) = driver++ .find_window(harness.pid as i64, "CuaTestHarness AppKit")++ .expect("find raw click receiver window");++ let (_, passed) = run_with_background_oracles(++ &mut driver,++ TargetWindow {++ pid: harness.pid,++ native_id: wid,++ },++ |driver| {++ let pre = snapshot_elements(driver, harness.pid, wid);++ assert!(pre.tree_text().contains("raw_ax_presses=0"));++ assert!(pre.tree_text().contains("raw_mouse_downs=0"));++ assert!(pre.tree_text().contains("raw_mouse_ups=0"));++ let (x, y, width, height) = element_pixel_frame(&pre, "btn-raw-hit-test");++++ let auto = driver.call(++ "click",++ serde_json::json!({++ "pid": harness.pid,++ "window_id": wid,++ "x": x + width / 2.0,++ "y": y + height / 2.0,++ "count": 1,++ "delivery_mode": "background"++ }),++ );++ assert!(!auto.is_error(), "default click failed: {}", auto.text());++ std::thread::sleep(Duration::from_millis(250));++ let after_auto = snapshot_elements(driver, harness.pid, wid);++ assert!(++ after_auto.tree_text().contains("raw_ax_presses=1")++ && after_auto.tree_text().contains("raw_mouse_downs=0")++ && after_auto.tree_text().contains("raw_mouse_ups=0"),++ "default count=1 click must preserve AX-first behavior:\n{}",++ after_auto.tree_text()++ );++++ let raw = driver.call(++ "click",++ serde_json::json!({++ "pid": harness.pid,++ "window_id": wid,++ "x": x + width / 2.0,++ "y": y + height / 2.0,++ "button": "left",++ "count": 1,++ "delivery_mode": "background",++ "click_mode": "raw"++ }),++ );++ assert!(!raw.is_error(), "raw click failed: {}", raw.text());++ std::thread::sleep(Duration::from_millis(250));++ let after_raw = snapshot_elements(driver, harness.pid, wid);++ assert!(++ after_raw.tree_text().contains("raw_ax_presses=1")++ && after_raw.tree_text().contains("raw_mouse_downs=1")++ && after_raw.tree_text().contains("raw_mouse_ups=1"),++ "raw count=1 click must deliver exactly one native pair without AX:\n{}",++ after_raw.tree_text()++ );++ },++ )++ .unwrap_or_else(|error| panic!("background desktop contract failed: {error}"));++ Observation::delivered_with_fixture_state(passed)++ });++}+++ #[test]+ #[ignore]+ fn harness_appkit_right_click_px_foreground() {+diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+index 7996734a9..69992dc6e 100644+--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs++++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+@@ -63,3 +63,23 @@ enum PixelActivationPolicy {++#[derive(Clone, Copy, Debug, PartialEq, Eq)]++enum ClickMode {++ Auto,++ Raw,++}++++impl ClickMode {++ fn parse(value: &str) -> Option<Self> {++ match value.trim().to_ascii_lowercase().as_str() {++ "" | "auto" => Some(Self::Auto),++ "raw" => Some(Self::Raw),++ _ => None,++ }++ }++++ fn is_raw(self) -> bool {++ self == Self::Raw++ }++}+++ #[derive(Clone, Copy, Debug)]+ struct SelectionPixelTarget {+ screen_x: f64,+@@ -196,7 +216,10 @@ fn def() -> &'static ToolDef {+ Pixel path: routes through the CGEvent left/right/middle mouse-button primitives. \+ AX path: \"right\" maps to AXShowMenu (same surface as the dedicated `right_click` \+ tool); \"middle\" has no AX equivalent and falls back to a pixel middle-click at the \+- element's center.\n\++ element's center. For a window-addressed background pixel left-click, \++ click_mode:\"raw\" bypasses the AX hit-test optimization and posts exactly one \++ routed mouse down/up pair. Keep the default click_mode:\"auto\" unless AXPress \++ was observed to return a hollow success.\n\+ action: press (default), show_menu, pick, confirm, cancel, open.\n\+ from_zoom: set true after a zoom call to auto-translate zoom-image pixel \+ coordinates to full-window space."+@@ -224,6 +247,11 @@ fn def() -> &'static ToolDef {+ "description": "Mouse button. Default: \"left\" — omit for legacy left-click behaviour. Pixel path uses the matching CGEvent primitive; AX path maps \"right\" to AXShowMenu and falls back to a pixel middle-click at the element's center for \"middle\"."+ },+ "count": { "type": "integer", "description": "Click count (pixel path only). Default 1." },++ "click_mode": {++ "type": "string",++ "enum": ["auto", "raw"],++ "description": "Pixel single-click delivery selector. Default \"auto\" preserves the AX-first background optimization. \"raw\" is opt-in and valid only for an exact pid + window_id background left-click with x/y, press action, count=1, and no modifiers; it bypasses AX hit-testing and uses the existing routed CGEvent path."++ },+ "modifier": {+ "type": "array",+ "items": { "type": "string" },+@@ -277,6 +305,24 @@ impl Tool for ClickTool {+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);++ let click_mode_value = args.str_or("click_mode", "auto");++ let click_mode = match ClickMode::parse(&click_mode_value) {++ Some(mode) => mode,++ None => {++ return ToolResult::error(format!(++ "click: unknown click_mode \"{}\" — expected auto or raw.",++ click_mode_value.trim().to_ascii_lowercase()++ ))++ .with_structured(serde_json::json!({ "code": "invalid_arguments" }));++ }++ };++ if click_mode.is_raw() && (!has_pid || !has_window_id || !has_xy) {++ return ToolResult::error(++ "click(click_mode=\"raw\") requires pixel coordinates x/y and an exact \++ pid + window_id target.",++ )++ .with_structured(serde_json::json!({ "code": "invalid_arguments" }));++ }+ let capture_id = args.opt_str("capture_id");+ if capture_id.is_some() && !has_xy {+ return ToolResult::error("click.capture_id requires pixel coordinates x and y.")+@@ -463,3 +509,22 @@ impl Tool for ClickTool {++ if click_mode.is_raw()++ && (element_guard.is_some()++ || args.get("target").is_some()++ || x.is_none()++ || y.is_none()++ || window_id.is_none()++ || !matches!(action.as_str(), "press" | "click")++ || button_str != "left"++ || count != 1++ || !modifiers.is_empty()++ || delivery_mode.is_foreground())++ {++ return ToolResult::error(++ "click(click_mode=\"raw\") requires a background pixel left-click with \++ exact pid + window_id, x/y, press action, count=1, and no modifiers.",++ )++ .with_structured(serde_json::json!({ "code": "invalid_arguments" }));++ }+++ // PID-routed key transitions can look correct for one AX poll and then+ // collapse to a plain click once AppKit resolves the gesture. Refuse+ // that false-success path. The explicit foreground rung uses physical+@@ -987,7 +1052,8 @@ impl Tool for ClickTool {+ // backend after resolving the requested screen point. This keeps+ // targeting (PX) orthogonal to delivery (AX) and avoids making a+ // Chromium/AppKit window key merely to satisfy first-mouse rules.+- if !delivery_mode.is_foreground()++ if click_mode == ClickMode::Auto++ && !delivery_mode.is_foreground()+ && window_id.is_some()+ && button_str == "left"+ && count == 1+@@ -1641,4 +1707,24 @@ mod tests {+ desc.contains("middle"),+ "description should mention middle button"+ );++ assert!(++ desc.contains("click_mode"),++ "description should mention the raw single-click selector"++ );++ }++++ #[test]++ fn schema_advertises_opt_in_raw_click_mode() {++ let d = def();++ let click_mode = &d.input_schema["properties"]["click_mode"];++ assert_eq!(click_mode["type"], "string");++ assert_eq!(click_mode["enum"], serde_json::json!(["auto", "raw"]));++ }++++ #[test]++ fn click_mode_preserves_auto_default_and_rejects_unknown_values() {++ assert_eq!(ClickMode::parse(""), Some(ClickMode::Auto));++ assert_eq!(ClickMode::parse("auto"), Some(ClickMode::Auto));++ assert_eq!(ClickMode::parse("RAW"), Some(ClickMode::Raw));++ assert_eq!(ClickMode::parse("semantic"), None);+ }+diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift+index 4621a4627..f8f0125ce 100644+--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift++++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift+@@ -55,6 +55,11 @@ let kMenuActionAID = "lbl-menu-action"+ let kScrollerAID = "scroll-tall"+ let kScrollOffsetAID = "lbl-scroll-offset"+ let kAccelCountAID = "lbl-accel-count"++let kRawClickTargetAID = "btn-raw-hit-test"++let kRawAxPressCountAID = "lbl-raw-ax-press-count"++let kRawMouseDownCountAID = "lbl-raw-mouse-down-count"++let kRawMouseUpCountAID = "lbl-raw-mouse-up-count"++let kRawClickTargetEnv = "CUA_APPKIT_RAW_CLICK_TARGET"+ let kScrollTopMarker = "SCROLL_TOP_MARKER_v1"+ let kScrollBottomMarker = "SCROLL_BOTTOM_MARKER_v1"+ let kExitButtonAID = "btn-exit"+@@ -130,6 +135,12 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewD+ let scrollOffsetLabel = NSTextField(labelWithString: "scroll_offset=0")+ let accelCountLabel = NSTextField(labelWithString: "accel_fired=0")+ var accelCount = 0++ let rawAxPressCountLabel = NSTextField(labelWithString: "raw_ax_presses=0")++ let rawMouseDownCountLabel = NSTextField(labelWithString: "raw_mouse_downs=0")++ let rawMouseUpCountLabel = NSTextField(labelWithString: "raw_mouse_ups=0")++ var rawAxPressCount = 0++ var rawMouseDownCount = 0++ var rawMouseUpCount = 0+ var keyMonitor: Any?+ // Opt-in jev-use task controls and their app-owned state file.+ let taskStatePath = ProcessInfo.processInfo.environment[kTaskStateEnv]+@@ -267,3 +278,28 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewD++ if ProcessInfo.processInfo.environment[kRawClickTargetEnv] == "1" {++ content.addArrangedSubview(sectionLabel("raw_click_target"))++ let rawTarget = HitTestOnlyView(harness: self)++ rawTarget.setAccessibilityIdentifier(kRawClickTargetAID)++ rawTarget.translatesAutoresizingMaskIntoConstraints = false++ rawAxPressCountLabel.setAccessibilityIdentifier(kRawAxPressCountAID)++ rawMouseDownCountLabel.setAccessibilityIdentifier(kRawMouseDownCountAID)++ rawMouseUpCountLabel.setAccessibilityIdentifier(kRawMouseUpCountAID)++ for label in [rawAxPressCountLabel, rawMouseDownCountLabel, rawMouseUpCountLabel] {++ label.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)++ }++ let rawRow = NSStackView()++ rawRow.orientation = .horizontal++ rawRow.spacing = 12++ rawRow.addArrangedSubview(rawTarget)++ rawRow.addArrangedSubview(rawAxPressCountLabel)++ rawRow.addArrangedSubview(rawMouseDownCountLabel)++ rawRow.addArrangedSubview(rawMouseUpCountLabel)++ NSLayoutConstraint.activate([++ rawTarget.widthAnchor.constraint(equalToConstant: 180),++ rawTarget.heightAnchor.constraint(equalToConstant: 48),++ ])++ content.addArrangedSubview(rawRow)++ }+++ // slider — NSSlider drives the `drag` / `set_value` tools (AXValue).+ content.addArrangedSubview(sectionLabel("slider"))+ let slider = NSSlider(value: 0, minValue: 0, maxValue: 100,+@@ -700,6 +736,18 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewD+ @objc func onClickTarget() { recordClick("click") }+ func clickTargetSawDouble() { recordClick("double_click") }+ func clickTargetSawRight() { recordClick("right_click") }++ func rawTargetSawAxPress() {++ rawAxPressCount += 1++ rawAxPressCountLabel.stringValue = "raw_ax_presses=\(rawAxPressCount)"++ }++ func rawTargetSawMouseDown() {++ rawMouseDownCount += 1++ rawMouseDownCountLabel.stringValue = "raw_mouse_downs=\(rawMouseDownCount)"++ }++ func rawTargetSawMouseUp() {++ rawMouseUpCount += 1++ rawMouseUpCountLabel.stringValue = "raw_mouse_ups=\(rawMouseUpCount)"++ }+ private func recordClick(_ action: String) {+ clicks += 1+ lastActionLabel.stringValue = "last_action=\(action)"+@@ -732,3 +780,56 @@ final class ClickTargetButton: NSButton {++final class HitTestOnlyView: NSView {++ weak var harness: HarnessWindowController?++++ init(harness: HarnessWindowController) {++ self.harness = harness++ super.init(frame: .zero)++ setAccessibilityElement(true)++ setAccessibilityRole(.button)++ setAccessibilityLabel("Raw hit-test button")++ }++++ required init?(coder: NSCoder) {++ fatalError("HitTestOnlyView does not support NSCoder")++ }++++ override func draw(_ dirtyRect: NSRect) {++ NSColor.systemBlue.setFill()++ NSBezierPath(roundedRect: bounds, xRadius: 6, yRadius: 6).fill()++ let text = "RAW HIT-TEST"++ let attributes: [NSAttributedString.Key: Any] = [++ .font: NSFont.boldSystemFont(ofSize: 14),++ .foregroundColor: NSColor.white,++ ]++ let size = text.size(withAttributes: attributes)++ text.draw(++ at: NSPoint(++ x: bounds.midX - size.width / 2,++ y: bounds.midY - size.height / 2++ ),++ withAttributes: attributes++ )++ }++++ override func hitTest(_ point: NSPoint) -> NSView? {++ bounds.contains(point) ? self : nil++ }++++ override func mouseDown(with event: NSEvent) {++ harness?.rawTargetSawMouseDown()++ }++++ override func mouseUp(with event: NSEvent) {++ harness?.rawTargetSawMouseUp()++ }++++ override func accessibilityPerformPress() -> Bool {++ harness?.rawTargetSawAxPress()++ return true++ }++++ override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }++}+++ // Opt-in windows for the persistent exact-window activation certification.+ // Ordinary harness launches remain byte-for-byte and behaviorally unchanged.+ final class BringToFrontMatrixWindows {diff --git a/desktop/cua-driver/sources.json b/desktop/cua-driver/sources.jsonindex 964ce1533..b2a643629 100644--- a/desktop/cua-driver/sources.json+++ b/desktop/cua-driver/sources.json@@ -4,6 +4,7 @@"version": "0.31.0","tag": "cua-driver-rs-v0.31.0","source_commit": "5272e492d61b96caf08e3bf434d91126c1f3dccc",+ "tool_snapshot_sha256": "4257e598f5a6ad7e632b48e96c03257d7c5ed618a93b72820b2d4fef0f196acf","license": "MIT","release_base_url": "https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.31.0","release_checksums_asset": "checksums.txt",@@ -11,6 +12,27 @@"macos": {"asset": "cua-driver-rs-0.31.0-darwin-universal-binary.tar.gz","sha256": "06cd80b153bdf046dc067fb593e0fc648e780afa37a902ca0515ac25f32f9a4f"+ },+ "source": {+ "archive": "cua-5272e492d61b96caf08e3bf434d91126c1f3dccc.tar.gz",+ "url": "https://codeload.github.com/trycua/cua/tar.gz/5272e492d61b96caf08e3bf434d91126c1f3dccc",+ "sha256": "8134337f2beb1854918d8bbc5aea22d7ae00de9838ce6426f275d7ba6734d191",+ "root": "cua-5272e492d61b96caf08e3bf434d91126c1f3dccc"+ },+ "patch": {+ "variant": "avibe-raw-single-click-v1",+ "file": "patches/0001-macos-raw-single-click.patch",+ "sha256": "45c28ca5fc737649b85ddebbb57ef837490b6349ecc1e0ac04782bfee58bdbe8",+ "contract": "click.click_mode=raw",+ "upstream_refs": [+ "https://github.com/trycua/cua/pull/3922",+ "https://github.com/trycua/cua/pull/3539",+ "https://github.com/trycua/cua/pull/4632",+ "https://github.com/trycua/cua/pull/4208",+ "https://github.com/trycua/cua/issues/4350",+ "https://github.com/trycua/cua/issues/2874"+ ],+ "upgrade_plan": "Drop this patch when an upstream release exposes an opt-in raw count=1 macOS click selector with the same default-compatible contract; otherwise rebase it onto the next pinned source commit and rerun the hit-test-only acceptance."}},"managed_policy": "policy.yaml",diff --git a/desktop/cua-driver/tools-v0.31.0.json b/desktop/cua-driver/tools-v0.31.0.jsonindex 0d7486dc2..1cd48e132 100644--- a/desktop/cua-driver/tools-v0.31.0.json+++ b/desktop/cua-driver/tools-v0.31.0.json@@ -1 +1 @@-{"driver_version":"0.31.0","schema_version":1,"source_tag":"cua-driver-rs-v0.31.0","tools":[{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["app.list"],"description":"List macOS apps — both currently running and installed-but-not-running — with per-app state flags:\n\n- running: is a process for this app live? (pid is 0 when false)\n- active: is it the system-frontmost app? (implies running)\n- launch_path: filesystem path to the `.app` bundle, when known. Pass this to `launch_app` to start the app cold.\n- kind: `\"desktop\"` for `.app` bundles on macOS.\n- last_used: RFC3339 timestamp from the bundle's filesystem mtime, when readable; otherwise null.\n\nStandalone running entries include only apps with NSApplicationActivationPolicyRegular — background helpers and system UI agents are filtered out. Installed apps resolve their running/pid state against all live processes by bundle identifier, so an installed app whose process runs as an accessory (LSUIElement / menu-bar apps, e.g. Cua Driver itself) still reports its live pid. Installed apps come from scanning /Applications, /Applications/Utilities, ~/Applications, /System/Applications, and /System/Applications/Utilities.\n\nUse this for \"is X installed?\" as well as \"is X running?\". For per-window state — on-screen, on-current-Space, minimized, window titles — call list_windows instead. For just opening an app — running or not — call launch_app({bundle_id: ...}) directly; list_apps is not a prerequisite.","inputSchema":{"additionalProperties":false,"properties":{"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"list_apps","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"apps":{"items":{"properties":{"active":{"type":"boolean"},"bundle_id":{"type":["string","null"]},"kind":{"type":["string","null"]},"last_used":{"type":["string","null"]},"launch_path":{"type":["string","null"]},"name":{"type":"string"},"pid":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["pid","name","running","active"],"type":"object"},"type":"array"}},"required":["apps"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["window.list"],"description":"List all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state.\n\nPer-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, space_ids, current_space_id (the active Space on that window's display), and on_current_space. The top-level current_space_id is WindowServer's main/global active Space and can differ from a record's current_space_id when displays use independent Spaces. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.","inputSchema":{"additionalProperties":false,"properties":{"on_screen_only":{"description":"When true, drop windows not on the current Space. Default false.","type":"boolean"},"pid":{"description":"Optional pid filter. When set, only this pid's windows are returned.","type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"list_windows","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"current_space_id":{"minimum":0,"type":["integer","null"]},"windows":{"items":{"properties":{"app_name":{"type":"string"},"bounds":{"properties":{"height":{"type":"number"},"width":{"type":"number"},"x":{"type":"number"},"y":{"type":"number"}},"required":["x","y","width","height"],"type":"object"},"current_space_id":{"minimum":0,"type":["integer","null"]},"is_on_screen":{"type":"boolean"},"layer":{"format":"int32","type":["integer","null"]},"minimized":{"type":["boolean","null"]},"on_current_space":{"type":["boolean","null"]},"pid":{"maximum":4294967295,"minimum":0,"type":["integer","null"]},"space_ids":{"items":{"minimum":0,"type":"integer"},"type":["array","null"]},"title":{"type":"string"},"window_id":{"minimum":0,"type":"integer"},"z_index":{"description":"Higher values are closer to the front. Null means the provider cannot observe stacking order; callers must not infer an order from array position or treat null as zero.","type":["integer","null"]}},"required":["window_id","pid","app_name","title","bounds","is_on_screen","z_index"],"type":"object"},"type":"array"}},"required":["windows"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":false,"readOnlyHint":true},"capabilities":["accessibility.window_state","accessibility.tree","accessibility.tree.structured","accessibility.tree.bounded","accessibility.element_tokens","screen.capture","screen.capture.window"],"description":"Walk a running app's AX tree and return BOTH a structured `elements` array (preferred) AND a Markdown rendering of the same tree (back-compat). Every actionable element is tagged with [element_index N] in the markdown and as `element_index` in the structured array; pass each element's `element_token` to click, type_text, press_key, etc.\n\nINVARIANT: call get_window_state once per turn per (pid, window_id) before any element action. The next snapshot of the window replaces this one, stales its element tokens, and lists the replaced ids in `invalidated_snapshot_ids`.\n\nPREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value` (the element's text/AXValue when present — use it to verify what a field holds), `actions` (names of AX actions exposed by the element, omitted when empty), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side.\n\nAlways returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces: Electron echo-confirms, Catalyst null values, virtualized off-viewport rows with `h:1` frames). You choose the modality at ACTION time, not here: an element ax action (pass `element_token` → the accessibility rung) or an element px action (pass `x`,`y` → the pixel rung, read straight off this screenshot). `capture_mode` is deprecated and ignored. Pass `include_screenshot:false` to skip the grab and get the tree only — the cheap path when you're just re-indexing before an element ax action.\n\nThe mirror image: pass `include_accessibility_tree:false` to SKIP the AX walk entirely (the expensive part, bounded by timeout_ms) and return just the screenshot plus window metadata — `window_bounds`, `screenshot_scale`, `screenshot_width`/`screenshot_height`, `app_name`, and `window_title` — the capture-only path for rendering a live window preview / picture-in-picture without paying for perception. Setting BOTH `include_accessibility_tree:false` and `include_screenshot:false` is an error (nothing to return). Optional `max_image_dimension` overrides the configured screenshot long-edge limit for this call; use 0 for native resolution. The legacy `max_dimension` remains a tighter cap for compatibility.\n\nThe snapshot is SCOPED to `window_id`: a window_id that no longer exists is refused with `window_id_not_found`, and one owned by another process is refused with `window_owner_pid_mismatch` naming the real `owner_pid` to retry with (macOS hosts a sandboxed app's Open/Save panel out-of-process, so its window belongs to the panel service, not the app). If the window is live under this pid but its accessibility surface can't be resolved, the tree comes back EMPTY with `degraded_reason: ax_window_unresolved` and the screenshot of the requested window; background input is refused until it resolves, so re-snapshot or act with `delivery_mode:\"foreground\"`. When that pid is an app still launching (its window exists before it answers accessibility), the walk first waits up to `timeout_ms` for it; if it never answers, the tree comes back EMPTY with `degraded_reason: ax_app_launching`, `truncated: true` and `truncation_reason: app_lookup_timeout`. A window on another Space still resolves by its exact CGWindowID. This tool never returns another surface's elements under your window_id. Before exposing a screenshot, its raw dimensions are validated as a coherent 1x/2x representation of the requested WindowServer bounds. `px_frame_mismatch` or `px_capture_unavailable` omits an unprovable screenshot/pixel frame instead of guessing a transform; the truthful AX payload remains available.\n\nOptional `query` projects both tree_markdown and structured `elements` to matching lines plus their ancestor chain (case-insensitive substring). The element_index values are unchanged, the complete snapshot remains actionable, and `element_count` continues to report its total size; `filtered_element_count` reports the projected response size.\n\nOptional `max_elements` / `max_depth` bound the AX walk to mitigate context-window blow-up on Electron / Obsidian / large web apps that produce 10k+ element trees. When applied, BOTH the markdown and the structured elements are truncated identically. Omit both for current default behaviour (≤2 000 elements, depth ≤25).","inputSchema":{"additionalProperties":false,"properties":{"capture_mode":{"description":"DEPRECATED and ignored. get_window_state always returns BOTH the element tree and a screenshot — ground on both. The modality is chosen at action time by how you address the target: an element ax action (element_token) or an element px action (x,y). Any value (including the old \"som\"/\"screenshot\" aliases) is accepted but has no effect.","enum":["ax","vision"],"type":"string"},"include_accessibility_tree":{"description":"Default true — walk the AX tree and return `elements` + `tree_markdown` alongside the screenshot. Set false to SKIP the AX walk entirely (the expensive part, bounded by timeout_ms) and return just the screenshot plus window metadata (bounds, scale, app_name, window_title) — the capture-only path for rendering a live window preview / picture-in-picture. Mirrors include_screenshot. Setting BOTH include_accessibility_tree:false AND include_screenshot:false is an error (nothing to return).","type":"boolean"},"include_screenshot":{"description":"Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return the tree only (the cheap path when you're just re-indexing before an element ax action; saves the image tokens + screen-grab latency). screenshot_out_file still forces a capture to disk.","type":"boolean"},"max_depth":{"description":"Cap on the AX-tree walk depth. Nodes whose rendered indent would exceed this are omitted. Omit for the default (25). Lower this for deep menu/Electron trees.","minimum":1,"type":"integer"},"max_dimension":{"description":"Optional cap on the returned screenshot's long edge, in pixels (aspect ratio preserved) — the cheap path for a small preview / thumbnail. Applied on top of the session/global max_image_dimension ceiling; the tighter of the two wins. Omit for the configured default.","minimum":1,"type":"integer"},"max_elements":{"description":"Cap on the total number of AX nodes walked. Truncates depth-first; markdown and structured elements truncate together. Omit for the default (2 000). Lower this for Electron / Obsidian / large web apps that produce 10k+ element trees and blow context windows.","minimum":1,"type":"integer"},"max_image_dimension":{"description":"Per-call override for the returned screenshot's long edge in pixels. An explicit value wins over the session/global setting; 0 returns native resolution. Omit to preserve configured behavior.","minimum":0,"type":"integer"},"pid":{"description":"Target process ID.","type":"integer"},"query":{"description":"Case-insensitive filter for tree_markdown and structured elements. Returns matching actionable rows plus their actionable ancestors without renumbering element_index values.","type":"string"},"screenshot_out_file":{"description":"When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output will contain screenshot_file_path instead.","type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"timeout_ms":{"default":1000,"description":"Wall-clock budget in milliseconds for the accessibility-tree walk (default 1000, min 100, max 120000). Bounds the WHOLE walk. When the budget runs out the tool returns the PARTIAL tree it has, flagged with `truncated: true`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `elements_complete: false`; retry with a larger value (e.g. 5000) or narrow with `query` / `max_depth`.","maximum":120000,"minimum":100,"type":"integer"},"window_id":{"description":"Target window ID from list_windows.","type":"integer"}},"required":["pid","window_id"],"type":"object"},"name":"get_window_state","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"app_name":{"type":["string","null"]},"degraded":{"type":["boolean","null"]},"degraded_reason":{"type":["string","null"]},"element_count":{"minimum":0,"type":["integer","null"]},"elements":{"items":{"properties":{"actions":{"items":{"type":"string"},"type":["array","null"]},"depth":{"minimum":0,"type":"integer"},"element_index":{"minimum":0,"type":"integer"},"element_token":{"type":["string","null"]},"enabled":{"type":["boolean","null"]},"frame":{"properties":{"h":{"type":"number"},"w":{"type":"number"},"x":{"type":"number"},"y":{"type":"number"}},"required":["x","y","w","h"],"type":["object","null"]},"in_web_content":{"type":["boolean","null"]},"label":{"type":["string","null"]},"max":{"type":["number","null"]},"min":{"type":["number","null"]},"parent_index":{"minimum":0,"type":["integer","null"]},"role":{"type":"string"},"selected":{"type":["boolean","null"]},"value":{"type":["string","null"]},"value_description":{"type":["string","null"]}},"required":["element_index","role","depth"],"type":"object"},"type":["array","null"]},"elements_complete":{"type":["boolean","null"]},"filtered_element_count":{"minimum":0,"type":["integer","null"]},"pid":{"minimum":0,"type":"integer"},"returned_element_count":{"minimum":0,"type":["integer","null"]},"screenshot_file_path":{"type":["string","null"]},"screenshot_frame_valid":{"type":["boolean","null"]},"screenshot_height":{"minimum":0,"type":["integer","null"]},"screenshot_mime_type":{"type":["string","null"]},"screenshot_scale":{"type":["number","null"]},"screenshot_width":{"minimum":0,"type":["integer","null"]},"snapshot_id":{"type":["string","null"]},"total_element_count":{"minimum":0,"type":["integer","null"]},"tree_markdown":{"type":["string","null"]},"truncated":{"type":["boolean","null"]},"truncation_reason":{"type":["string","null"]},"window_bounds":{"properties":{"height":{"type":"number"},"width":{"type":"number"},"x":{"type":"number"},"y":{"type":"number"}},"required":["x","y","width","height"],"type":["object","null"]},"window_id":{"minimum":0,"type":"integer"},"window_title":{"type":["string","null"]}},"required":["pid","window_id"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r3","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":false,"readOnlyHint":true},"capabilities":["state.verify","state.verify.window","state.verify.element","state.observe.visual_evidence"],"description":"Deterministically verify bounded predicates against one exact window. The driver evaluates structured window/accessibility state and may return the final screenshot as uninterpreted visual evidence for a multimodal caller. Predicate results are satisfied, unsatisfied, or unknown; unknown never implies success. Accessibility projections are conservative: absence remains unknown unless the observed search domain is proven exhaustive.","inputSchema":{"additionalProperties":false,"properties":{"expect":{"description":"One to eight predicates, combined with logical AND.","items":{"additionalProperties":false,"properties":{"element":{"additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"exists":{"description":"Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.","type":"boolean"},"selected":{"type":["boolean","null"]},"selector":{"additionalProperties":false,"properties":{"label_contains":{"minLength":1,"type":"string"},"role":{"minLength":1,"type":"string"}},"required":[],"type":"object"},"value_equals":{"type":["string","null"]}},"required":["selector"],"type":["object","null"]},"window":{"additionalProperties":false,"properties":{"bounds":{"additionalProperties":false,"properties":{"height":{"type":"number"},"tolerance_px":{"maximum":100,"minimum":0,"type":"number"},"width":{"type":"number"},"x":{"type":"number"},"y":{"type":"number"}},"required":["x","y","width","height"],"type":["object","null"]},"exists":{"type":["boolean","null"]}},"type":["object","null"]}},"required":[],"type":"object"},"maxItems":8,"minItems":1,"type":"array"},"include_screenshot":{"description":"Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.","type":["boolean","null"]},"pid":{"description":"Exact process whose window may be observed.","minimum":1,"type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session. This\nfield never selects capture modality or authorization.","type":"string"},"stable_samples":{"default":2,"description":"Consecutive satisfied samples required before returning success.","maximum":5,"minimum":1,"type":"integer"},"timeout_ms":{"default":5000,"description":"Bounded wait. Zero performs one sample.","maximum":10000,"minimum":0,"type":"integer"},"window_id":{"description":"Exact native window identifier.","type":"integer"}},"required":["pid","window_id","expect"],"type":"object"},"name":"verify_state","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"elapsed_ms":{"minimum":0,"type":"integer"},"predicates":{"items":{"properties":{"index":{"minimum":0,"type":"integer"},"observed_json":{"anyOf":[{"type":"string"},{"type":"null"}]},"status":{"enum":["satisfied","unsatisfied","unknown"],"type":"string"},"unknown_reason":{"anyOf":[{"enum":["invalid_predicate","unsupported_predicate","untrusted_source","multi_match","target_missing","observation_unavailable","stability_unproven"],"type":"string"},{"type":"null"}]}},"required":["index","status","unknown_reason","observed_json"],"type":"object"},"type":"array"},"samples":{"minimum":0,"type":"integer"},"stable":{"type":"boolean"},"status":{"enum":["satisfied","unsatisfied","unknown"],"type":"string"}},"required":["status","stable","elapsed_ms","samples","predicates"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":true,"readOnlyHint":false},"capabilities":["app.launch"],"description":"Launch a macOS app in the background — the target does NOT come to the foreground.\n\nProvide either `bundle_id` (preferred — unambiguous, e.g. `com.apple.calculator`) or `name` (e.g. \"Calculator\"). If both are given, bundle_id wins.\n\nOptional `urls` are handed to the app as open targets — for Finder, pass a folder path to open a backgrounded Finder window there.\n\nBrowser DevTools setup belongs to `browser_prepare`, which can prove that a separate isolated profile is driver-owned before enabling CDP.\n\nOptional `webkit_inspector_port`: opens a WebKit inspector server on the specified port (sets WEBKIT_INSPECTOR_SERVER=127.0.0.1:N + TAURI_WEBVIEW_AUTOMATION=1). Use this for Tauri/WebKit-based apps.\n\nOptional `creates_new_application_instance`: when true, forces a new app instance even if one is already running (passes -n to open). Reach for this when another agent or session may drive the SAME app concurrently — it returns a fresh pid + window so each session acts on its own isolated window instead of clobbering one shared instance. Without it, single-instance apps (Calculator, many utilities) hand every caller the same window, so two sessions fight over it.\n\nOptional `additional_arguments`: extra argv strings appended after --args.\n\nReturns the launched app's pid, bundle_id, name, and a `windows` array (same shape as `list_windows`) so callers can skip an extra round-trip before `get_window_state(pid, window_id)`. `launch_state` distinguishes whether the request was sent, the process is running, and a window is ready. When the focus-steal belt-and-braces demotion check ran (target pid ≠ prior frontmost), the response also includes `self_activation_suppressed: bool` — true if focus stayed with the prior frontmost, false if the launched app held focus despite the re-demote attempt.","inputSchema":{"additionalProperties":false,"properties":{"additional_arguments":{"description":"Extra arguments appended after --args when launching.","items":{"type":"string"},"type":"array"},"bundle_id":{"description":"App bundle identifier, e.g. com.apple.calculator. Preferred over name.","type":"string"},"creates_new_application_instance":{"description":"When true, force a new app instance even if already running (open -n). Use for concurrent multi-agent/multi-session work so each session gets an isolated instance + window instead of sharing one — on single-instance apps (e.g. Calculator) every caller otherwise gets the same window and the sessions clobber each other.","type":"boolean"},"name":{"description":"App display name. Used only when bundle_id is absent.","type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"urls":{"description":"Optional file paths or URLs to open with the app (e.g. a folder path for Finder).","items":{"type":"string"},"type":"array"},"webkit_inspector_port":{"description":"Open a WebKit inspector server on this port (sets WEBKIT_INSPECTOR_SERVER env var).","type":"integer"}},"type":"object"},"name":"launch_app","risk":{"class":"r1","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["app.kill"],"description":"Force-terminate a process by pid (kill -9 equivalent on macOS / Linux; taskkill /F equivalent on Windows). Use as escalation when the cooperative close path (hotkey cmd+q on macOS, click-the-X on Windows) failed to make the process exit. Unsaved state is lost — prefer the cooperative path first.","inputSchema":{"additionalProperties":false,"properties":{"pid":{"description":"PID of the process to terminate.","type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"required":["pid"],"type":"object"},"name":"kill_app","risk":{"class":"r3","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["window.frame.set"],"description":"Set one exact top-level window's frame in the desktop-coordinate space reported by list_windows and verify the resulting geometry through an independent readback.","inputSchema":{"additionalProperties":false,"properties":{"height":{"minimum":1,"type":"number"},"pid":{"minimum":1,"type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"width":{"minimum":1,"type":"number"},"window_id":{"minimum":1,"type":"integer"},"x":{"type":"number"},"y":{"type":"number"}},"required":["pid","window_id","x","y","width","height"],"type":"object"},"name":"set_window_frame","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.click","input.pointer.click.left","accessibility.element_tokens","input.delivery_mode"],"description":"Click against a target pid. **Prefer `element_token` over pixel coordinates** — the token works on backgrounded / minimized / hidden / off-Space windows, identifies one exact snapshot element, and tells you what you're clicking via the cached element's role + label. Reach for `x, y` only when the target is a canvas / video / WebGL / custom-drawn surface that doesn't appear in the AX tree.\n\nTwo addressing modes:\n\n- element_token (from get_window_state): AX action path. Works on backgrounded/hidden windows. No cursor move, no focus steal. The snapshot cache is scoped per (pid, window_id) and is replaced by the next snapshot of the same window — re-snapshot every turn before clicking.\n\n- x, y (window-local screenshot pixels, top-left origin of the PNG returned by get_window_state): CGEvent path. Synthesizes mouse events and posts to pid. Use modifier for cmd/shift/option/ctrl. Needs a visible on-screen window to anchor the conversion.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat — omit it and you get the legacy left-click behaviour. Pixel path: routes through the CGEvent left/right/middle mouse-button primitives. AX path: \"right\" maps to AXShowMenu (same surface as the dedicated `right_click` tool); \"middle\" has no AX equivalent and falls back to a pixel middle-click at the element's center.\naction: press (default), show_menu, pick, confirm, cancel, open.\nfrom_zoom: set true after a zoom call to auto-translate zoom-image pixel coordinates to full-window space.","inputSchema":{"additionalProperties":false,"properties":{"action":{"description":"AX action: press, show_menu, pick, confirm, cancel, open.","type":"string"},"button":{"description":"Mouse button. Default: \"left\" — omit for legacy left-click behaviour. Pixel path uses the matching CGEvent primitive; AX path maps \"right\" to AXShowMenu and falls back to a pixel middle-click at the element's center for \"middle\".","enum":["left","right","middle"],"type":"string"},"capture_id":{"description":"Optional immutable source capture ID returned by get_window_state or get_desktop_state. With x,y, Driver atomically admits and consumes that exact capture before dispatch; stale, mismatched, or out-of-bounds captures are refused without fallback.","type":"string"},"count":{"description":"Click count (pixel path only). Default 1.","type":"integer"},"debug_image_out":{"description":"Optional file path. When set on a pixel-addressed click, captures a fresh screenshot, draws a red crosshair at (x, y), and writes the PNG. Use to verify coordinate spaces. Requires window_id; incompatible with from_zoom.","type":"string"},"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": perform the AX action or post the CGEvent without fronting. \"foreground\": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. Modified clicks require \"foreground\" so macOS observes physical modifier-key state. A generic click has no independent postcondition read-back, except selection of list-like AX rows whose AXSelected state can be confirmed; otherwise confirm the effect from a fresh state snapshot. Use the agent loop: background AX (element_token) → snapshot → background pixel (x/y) → snapshot → delivery_mode:\"foreground\".","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"from_zoom":{"description":"When true, x and y are in the last zoom image for this pid; driver translates back to full-window coordinates.","type":"boolean"},"modifier":{"description":"Modifier keys: cmd, shift, option/alt, ctrl.","items":{"type":"string"},"type":"array"},"pid":{"description":"Target process ID.","type":"integer"},"scope":{"description":"Coordinate frame for a windowless screen-absolute click (default \"window\"). Pass \"desktop\" when sending x,y with NO pid/window_id — the coordinates are then true screen pixels (read from get_desktop_state with scope=\"desktop\"). Per-call; not a setting.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},"window_id":{"description":"Target window ID. Omit when element_token is supplied (the token carries it).","type":"integer"},"x":{"description":"X in screenshot pixels. A window target uses the get_window_state PNG; a desktop target uses the native get_desktop_state PNG. The driver reverses Retina backing scale and any window-image downscale.","type":"number"},"y":{"description":"Y in screenshot pixels from the image selected by target.","type":"number"}},"required":[],"type":"object"},"name":"click","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.click","input.pointer.click.left","input.pointer.click.double","accessibility.element_tokens","input.delivery_mode"],"description":"Double-click at (x, y) or on an AX element identified by element_token.\n\nAX path (element_token provided): performs `AXOpen` when the element advertises it (Finder items, openable list rows/cells); otherwise resolves the element's on-screen center and falls back to a pixel double-click there.\n\nPixel path (x, y provided): two down/up pairs ~80 ms apart at the given coordinates.","inputSchema":{"additionalProperties":false,"properties":{"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"pid":{"type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"window_id":{"description":"CGWindowID. Omit when element_token is supplied (the token carries it).","type":"integer"},"x":{"description":"Screen X coordinate (pixel path).","type":"number"},"y":{"description":"Screen Y coordinate (pixel path).","type":"number"}},"required":["pid"],"type":"object"},"name":"double_click","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.click","input.pointer.click.right","accessibility.element_tokens","input.delivery_mode"],"description":"Right-click against a target pid. Two addressing modes:\n\n- `element_token` (from the last `get_window_state` snapshot) — performs `AXShowMenu` on the cached element. Pure AX RPC, works on backgrounded / hidden windows, no cursor move or focus steal. Requires a prior `get_window_state(pid, window_id)` in this turn.\n\n- `x`, `y` — synthesizes `rightMouseDown` / `rightMouseUp` CGEvent pair posted to the pid. Driver converts image-pixel → screen-point internally. `modifier` forces the CGEvent path (AX actions don't propagate modifier keys).\n\nExactly one of `element_token` or (`x` AND `y`) must be provided. `pid` always required.","inputSchema":{"additionalProperties":false,"properties":{"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"modifier":{"description":"Modifier keys held during the right-click: cmd/shift/option/ctrl. Pixel path only.","items":{"type":"string"},"type":"array"},"pid":{"description":"Target process ID.","type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"window_id":{"description":"CGWindowID. Omit when element_token is supplied (the token carries it).","type":"integer"},"x":{"description":"X in window-local screenshot pixels. Must be provided together with y.","type":"number"},"y":{"description":"Y in window-local screenshot pixels. Must be provided together with x.","type":"number"}},"required":["pid"],"type":"object"},"name":"right_click","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.drag","input.delivery_mode"],"description":"Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels — the same space get_window_state returns. Top-left origin of the target's window.\n\nUse for: marquee/lasso selection, drag-and-drop, resizing via a handle, scrubbing a slider, repositioning a panel.\n\n`duration_ms` (default 500) is the wall-clock budget for the path between mouse-down and mouse-up; `steps` (default 20) is the number of intermediate mouseDragged events linearly interpolated along the path. Increase both for slower, more human drags; decrease for snap gestures.\n\n`modifier` keys (cmd/shift/option/ctrl) are held across the entire gesture.\n\nWhen `from_zoom` is true, coordinates are in the last zoom image for this pid; the driver maps them back to window coordinates before dispatching.","inputSchema":{"additionalProperties":false,"properties":{"button":{"description":"Mouse button used for the drag. Default: left.","enum":["left","right","middle"],"type":"string"},"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"duration_ms":{"description":"Wall-clock duration of the drag path between mouseDown and mouseUp. Default: 500.","maximum":10000,"minimum":0,"type":"integer"},"from_x":{"description":"Drag-start X in window-local screenshot pixels. Top-left origin.","type":"number"},"from_y":{"description":"Drag-start Y in window-local screenshot pixels. Top-left origin.","type":"number"},"from_zoom":{"description":"When true, coordinates are in the last zoom image for this pid; driver maps back to window coordinates.","type":"boolean"},"modifier":{"description":"Modifier keys held across the entire gesture: cmd/shift/option/ctrl.","items":{"type":"string"},"type":"array"},"pid":{"description":"Target process ID.","type":"integer"},"scope":{"default":"window","description":"Use desktop with no pid/window_id for native get_desktop_state screenshot coordinates.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"steps":{"description":"Number of intermediate mouseDragged events linearly interpolated along the path. Default: 20.","maximum":200,"minimum":1,"type":"integer"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"to_x":{"description":"Drag-end X in window-local screenshot pixels.","type":"number"},"to_y":{"description":"Drag-end Y in window-local screenshot pixels.","type":"number"},"window_id":{"description":"CGWindowID for the window the pixel coordinates were measured against. Optional only when pid owns exactly one eligible top-level window; otherwise the action refuses with ambiguous_window_target.","type":"integer"}},"required":["from_x","from_y","to_x","to_y"],"type":"object"},"name":"drag","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.keyboard.type","input.keyboard.type.terminal_safe","accessibility.element_tokens","input.delivery_mode"],"description":"Insert text into the target pid via `AXSetAttribute(kAXSelectedText)`. Works for standard Cocoa text fields and text views. No keystrokes are synthesized — special keys (Return / Escape / arrows) go through `press_key` / `hotkey`. For Chromium / Electron inputs that don't implement `kAXSelectedText`, the tool falls back to CGEvent character synthesis automatically when the estimated route stays within the daemon transport budget. Longer synthesized routes are refused before character events and return a safe chunk size; one-call AX insertion remains uncapped.\n\nOptional `element_token` (from the last `get_window_state` snapshot) directs the write to a specific field. Without `element_token`, the write goes to the pid's currently focused element.\n\nWEB CONTENT (Chromium/WebKit/Electron — browser tabs, Slack, VS Code, X's compose box): AXValue is not independent proof that the renderer/DOM observed an AX write or synthesized keystrokes. The driver detects this at the element level (an AXWebArea ancestor) and refuses to trust AXValue-only read-back there. Electron AX targets that are web content or cannot be proven native refuse background delivery before mutation because the AX route cannot establish exact renderer focus; use the px form or explicit foreground delivery. Other web-content paths return effect:\"unverifiable\" + escalation, never a false \"confirmed\" (a browser's own native address bar/toolbar stays trusted). For a browser TAB the reliable path is the `page` tool (drives the DOM via CDP); for an embedded web view use this tool's px form: pass x,y (no element_token) to pixel-click the field then type, in one call. NOTE: a px focus-click won't reliably open+focus a CLOSED control; AX-press to open/activate it first (works in the background), then px-type. Always confirm via the screenshot; if px-background still drops, escalate to delivery_mode:\"foreground\".","inputSchema":{"additionalProperties":false,"properties":{"delay_ms":{"description":"Milliseconds between characters in the CGEvent fallback path. Default 30. Ignored when the AX path succeeds.","maximum":200,"minimum":0,"type":"integer"},"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": AX insert, then CGEvent keystrokes if needed — no focus steal; native controls can be confirmed via AXValue read-back, while web-content writes remain effect:\"unverifiable\". \"foreground\": briefly front the window, type, restore the prior frontmost — the explicit last resort for focus-sensitive surfaces (e.g. WhatsApp/Catalyst) where background keystrokes don't land. Re-call with \"foreground\" when a background attempt remains unverifiable and a fresh snapshot shows the text did not appear.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"pid":{"description":"Target process ID.","type":"integer"},"scope":{"default":"window","description":"Use desktop with no pid/window_id to type into the frontmost application.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"text":{"description":"Text to insert at the target's cursor.","type":"string"},"window_id":{"description":"CGWindowID. Omit when element_token is supplied (the token carries it).","type":"integer"},"x":{"description":"Screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the AX path can't reach. Read straight off the get_window_state PNG, same convention as click.","type":"number"},"y":{"description":"Screenshot-pixel Y of the field (see x).","type":"number"}},"required":["text"],"type":"object"},"name":"type_text","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.keyboard.press","accessibility.element_tokens","input.delivery_mode"],"description":"Press and release a single key. Follows the same `delivery_mode` ladder as click/type_text — it does NOT raise the window by default:\n• `background` (default): post to the pid WITHOUT fronting/raising — the auth-message path (Chromium-safe). With element_token it focuses that AX element first. `window_id` only targets; it does not raise.\n• `foreground`: guard and briefly front the exact window, focus an addressed AX element when supplied, send a genuine HID key transition so Chromium content, inline editors, and native menu equivalents receive it, then restore prior frontmost. Requires window_id.\n\nA key press is confirmed only when a bounded native AX value/selection read-back changes on the same control. Otherwise a successfully attempted post remains effect:\"unverifiable\" without implying delivery failure or recommending foreground. Key names: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, plus any letter or digit. Modifiers array: cmd, shift, option/alt, ctrl, fn.","inputSchema":{"additionalProperties":false,"properties":{"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"key":{"description":"Key name: return, tab, escape, up, down, etc.","type":"string"},"modifiers":{"description":"Modifier keys: cmd, shift, option/alt, ctrl, fn.","items":{"type":"string"},"type":"array"},"pid":{"type":"integer"},"scope":{"default":"window","description":"Use desktop with no pid/window_id to send the key to the frontmost application.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"window_id":{"description":"Target window. Required for delivery_mode:\"foreground\". Does NOT itself raise the window — raising is gated on delivery_mode.","type":"integer"},"x":{"description":"Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the AX path can't focus. Pass with y, no element_token.","type":"number"},"y":{"description":"Screenshot-pixel Y (see x).","type":"number"}},"required":["key"],"type":"object"},"name":"press_key","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.keyboard.hotkey","input.delivery_mode"],"description":"Press a key combination — e.g. `[\"cmd\", \"c\"]` for Copy, `[\"cmd\", \"shift\", \"4\"]` for screenshot selection. Follows the same `delivery_mode` ladder as click/type_text — it does NOT raise the window by default:\n• `background` (default): post the combo to the target pid WITHOUT fronting or raising it — uses the macOS 14+ auth-message envelope so Chromium/Electron accept it as trusted live input. With an AX target, focus that exact element first. No top-level focus steal. `window_id` here only targets the combo; it does not raise.\n• `foreground`: briefly front the window (NSMenu path, < 1 ms via SLPSSetFrontProcessWithOptions) so native menu key-equivalents (Cmd+Z, Cmd+W) dispatch, then restore the prior frontmost — the explicit escalation for menu-bar shortcuts on non-Chromium apps that ignore a background combo. With an AX target or x,y, the focused field receives the chord through the foreground HID queue (needed by native Chromium fields such as the omnibox). Requires window_id.\n\nA combo is never driver-verifiable (no read-back) → effect:\"unverifiable\"; confirm via screenshot. NOTE: a keyboard combo does NOT focus a text field — to type into a backgrounded Electron input, establish real renderer focus with a PIXEL click first, then `type_text`. If an app only accepts paste, call `clipboard_write`, then `clipboard_read` and verify its types (and text when applicable) before selecting or replacing editor content; only then send Cmd+V.\n\nRecognized modifiers: cmd/command, shift, option/alt, ctrl/control, fn. Non-modifier keys use the same vocabulary as `press_key`. Order: modifiers first, one non-modifier last.","inputSchema":{"additionalProperties":false,"properties":{"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"keys":{"description":"Modifier(s) and one non-modifier key, e.g. [\"cmd\", \"c\"].","items":{"type":"string"},"minItems":2,"type":"array"},"pid":{"description":"Target process ID.","type":"integer"},"scope":{"default":"window","description":"Use desktop with no pid/window_id to send the chord to the frontmost application.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"window_id":{"description":"Target window. Required for delivery_mode:\"foreground\" (the NSMenu activation needs a window). Does NOT itself raise the window — raising is gated on delivery_mode.","type":"integer"},"x":{"description":"Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the combo (so e.g. Cmd+V pastes into that field). Pass with y. Use for Chromium/Electron surfaces the background combo can't reach.","type":"number"},"y":{"description":"Screenshot-pixel Y (see x).","type":"number"}},"required":["keys"],"type":"object"},"name":"hotkey","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":true,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.keyboard.type","accessibility.element_tokens"],"description":"Set a value on a UI element. Two modes depending on element role:\n\n- **AXPopUpButton / select dropdown**: finds the child option whose title or value matches `value` (case-insensitive) and AXPresses it directly — the native macOS popup menu is never opened, so focus is never stolen. Use this for HTML <select> elements in Safari or any native NSPopUpButton.\n\n- **All other elements**: writes AXValue directly (sliders, steppers, date pickers, native text fields that expose settable AXValue).\n\nFor free-form text entry into web inputs, prefer `type_text_chars` which synthesises key events — AXValue writes are ignored by WebKit.","inputSchema":{"additionalProperties":false,"properties":{"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"pid":{"type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"value":{"description":"New value. AX will coerce to the element's native type.","type":"string"},"window_id":{"description":"CGWindowID. Omit when element_token is supplied (the token carries it).","type":"integer"}},"required":["pid","value"],"type":"object"},"name":"set_value","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.scroll","accessibility.element_tokens","input.delivery_mode"],"description":"Scroll the target pid. Two paths, picked by how you address the scroll:\n\n• **Targeted wheel path** — when you pass a target, either `element_token` (preferred) or window-local `x, y` pixels: the driver synthesizes a real mouse-wheel event (CGEventCreateScrollWheelEvent, at that screen point. The renderer hit-tests the wheel at the cursor, so the scroll lands on whatever element is under the point — exactly like physically rolling the wheel over it. This is the ONLY way to scroll a nested `overflow:auto` region (e.g. a scrollable <div> with no tabindex): such regions never take keyboard focus, so the keystroke path below no-ops on them. Use this for inner/nested scrollers in web views.\n\n• **Keystroke path (focused region)** — when you pass NO target (just pid + direction): synthesizes PageDown/PageUp (by='page') or Down/Up arrows (by='line'); horizontal uses Left/Right arrows. Drives the focused / page scroller only.\n\nMapping: by='page' → larger step; by='line' → smaller step; amount = number of wheel notches (targeted path) or keystroke repetitions (keystroke path).","inputSchema":{"additionalProperties":false,"properties":{"amount":{"description":"Pixel-wheel path: number of wheel notches. Keystroke path: number of keystroke repetitions. Larger requests are clamped to the maximum. Default: 3.","maximum":50,"minimum":1,"type":"integer"},"by":{"description":"Scroll granularity. Default: line.","enum":["line","page"],"type":"string"},"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"direction":{"description":"Scroll direction.","enum":["up","down","left","right"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"pid":{"type":"integer"},"scope":{"default":"window","description":"Use desktop with x,y and no pid/window_id for native get_desktop_state screenshot coordinates.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"window_id":{"type":"integer"},"x":{"description":"Window-local screenshot X (top-left origin of the PNG from get_window_state). With `y`, routes through the pixel-wheel path at this point — use for a scrollable surface that isn't in the AX tree. Requires window_id to anchor the window→screen conversion.","type":"number"},"y":{"description":"Window-local screenshot Y. See `x`.","type":"number"}},"required":["direction"],"type":"object"},"name":"scroll","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":false,"readOnlyHint":true},"capabilities":["clipboard.read","clipboard.types"],"description":"List available system clipboard types and optionally return privacy-sensitive plain text. Clipboard content is never retained in telemetry.","inputSchema":{"additionalProperties":false,"properties":{"include_text":{"default":false,"description":"Return plain-text clipboard content in addition to the available types.\nClipboard content is privacy-sensitive and is never retained in telemetry.","type":"boolean"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"required":[],"type":"object"},"name":"clipboard_read","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"content_redacted_from_telemetry":{"type":"boolean"},"privacy_sensitive":{"type":"boolean"},"supported":{"type":"boolean"},"text":{"anyOf":[{"type":"string"},{"type":"null"}]},"types":{"items":{"type":"string"},"type":"array"}},"required":["supported","types","text","privacy_sensitive","content_redacted_from_telemetry"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["clipboard.write","clipboard.write.text","clipboard.write.image","clipboard.write.file_url","clipboard.types"],"description":"Replace the system clipboard with exactly one value: plain text, an image from an absolute local path, or a file URL from an absolute local path. Returns the available types for read-back before paste.","inputSchema":{"additionalProperties":false,"properties":{"file_path":{"description":"Absolute path to a local file to place on the clipboard as a file URL.","type":"string"},"image_path":{"description":"Absolute path to a local image to place on the clipboard.","type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"text":{"description":"Plain text to place on the clipboard.","type":"string"}},"required":[],"type":"object"},"name":"clipboard_write","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"content_redacted_from_telemetry":{"type":"boolean"},"privacy_sensitive":{"type":"boolean"},"supported":{"type":"boolean"},"types":{"items":{"type":"string"},"type":"array"},"written_type":{"type":"string"}},"required":["supported","written_type","types","privacy_sensitive","content_redacted_from_telemetry"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["screen.dimensions"],"description":"Return the logical size of the main display in points plus its backing scale factor. Agents click in points; Retina displays have scale_factor 2.0. Requires no TCC permissions.","inputSchema":{"additionalProperties":false,"properties":{"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"get_screen_size","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"height":{"type":"number"},"scale_factor":{"type":"number"},"width":{"type":"number"}},"required":["width","height","scale_factor"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":false,"readOnlyHint":true},"capabilities":["screen.capture","screen.dimensions"],"description":"Capture the full display in true screen pixels, full size unless `max_image_dimension` caps it. Use its PNG as the coordinate source for actions whose target is {kind:\"desktop\",display_id:\"primary\"}. Returns the true screen size and backing scale factor. Vision-only: no AX tree walk.","inputSchema":{"additionalProperties":false,"properties":{"max_image_dimension":{"description":"Optional long-edge cap for the returned PNG, in pixels (aspect ratio preserved). Omitted or 0 returns the full-size capture. When the cap downsizes the image, the response reports `screenshot_original_width/height`, and x/y read off it for this session's later scope:\"desktop\" actions (or passed with its `capture_id`) are mapped back to the full-size frame automatically.","minimum":0,"type":"integer"},"screenshot_out_file":{"description":"Write PNG here instead of base64.","type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"get_desktop_state","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"agent_overlay_capture":{"additionalProperties":false,"properties":{"method":{"type":["string","null"]},"reason":{"type":["string","null"]},"status":{"oneOf":[{"const":"excluded","type":"string"},{"const":"not_present","type":"string"},{"const":"not_excluded","type":"string"}]}},"required":["status"],"type":["object","null"]},"display":{"type":"string"},"platform":{"enum":["macos","linux","windows"],"type":"string"},"scale_factor":{"type":"number"},"screen_height":{"type":"integer"},"screen_width":{"type":"integer"},"screenshot_file_path":{"type":"string"},"screenshot_height":{"type":"integer"},"screenshot_mime_type":{"const":"image/png"},"screenshot_width":{"type":"integer"}},"required":["platform","display","screenshot_width","screenshot_height","screen_width","screen_height","scale_factor","screenshot_mime_type"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r3","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["screen.cursor.position"],"description":"Return the current mouse cursor position in screen points (origin top-left).","inputSchema":{"additionalProperties":false,"properties":{"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"get_cursor_position","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"available":{"type":"boolean"},"source":{"type":"string"},"x":{"type":"number"},"y":{"type":"number"}},"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":[],"description":"Single-call end-to-end driver diagnostics. Designed to let downstream consumers ship one stable call instead of stitching together check_permissions, doctor, version, bundle attribution, and platform capability status. On macOS, prompt-capable direct capture is deliberately skipped; use `cua-driver permissions grant` to verify it explicitly. cua-driver owns the health model; consumers stay thin.\n\nInput — all optional:\n {\n \"include\": [\"<check_name>\", ...], // run only these\n \"skip\": [\"<check_name>\", ...] // skip these\n }\nIf both are given, `include` wins.\n\nCanonical check names:\n macOS : binary_version, platform_supported, session_active,\n bundle_identity, tcc_accessibility, tcc_screen_recording,\n ax_capability, screen_capture_capability\n Windows: binary_version, platform_supported, session_active,\n ax_capability (via UIA), screen_capture_capability (via DXGI)\n Linux : binary_version, platform_supported, session_active,\n ax_capability (via AT-SPI), screen_capture_capability (via X11)\n\nOutput — stable contract, schema_version=\"1\":\n {\n \"schema_version\": \"1\",\n \"platform\": \"darwin\" | \"win32\" | \"linux\",\n \"driver_version\": \"<semver>\",\n \"overall\": \"ok\" | \"degraded\" | \"failed\",\n \"checks\": [\n {\n \"name\": \"<one of the canonical names above>\",\n \"status\": \"pass\" | \"fail\" | \"skip\",\n \"message\": \"<one-line summary, always present>\",\n \"hint\": \"<remediation step, present when status=fail>\",\n \"data\": { /* check-specific structured fields */ }\n },\n ...\n ]\n }\n\n`overall` rules:\n - `ok` — every non-skipped check passes\n - `degraded` — at least one non-core check fails (binary is still usable)\n - `failed` — any core check fails (binary_version, platform_supported, session_active)\n\nStability: schema_version=\"1\" is the contract. Future breaking changes will be `\"2\"`. Adding new check names under the same schema_version is non-breaking; consumers must tolerate unknown check names.","inputSchema":{"additionalProperties":false,"properties":{"include":{"description":"Only run these checks (canonical names). Wins over `skip`.","items":{"type":"string"},"type":"array"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"skip":{"description":"Skip these checks (canonical names). Ignored when `include` is set.","items":{"type":"string"},"type":"array"}},"type":"object"},"name":"health_report","risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["accessibility.tree","accessibility.tree.structured"],"description":"Return a lightweight snapshot of the desktop: running regular apps and on-screen visible windows with their bounds, z-order, and owner pid.\n\nFor the full AX subtree of a single window (with interactive element indices you can click by), use `get_window_state` instead — that's the heavy per-window tool. This one is a fast discovery read that needs no TCC grants.","inputSchema":{"additionalProperties":false,"properties":{"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"get_accessibility_tree","risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["screen.capture","screen.capture.window","screen.capture.region"],"description":"Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in screenshot pixel coordinates, with 20% padding added on each side. The output image is at most 500 px wide.\n\nAfter a zoom, pass `from_zoom=true` to click/type_text to auto-translate coordinates back to full-window space. Coordinate actions return `screenshot_context_missing` when no current snapshot contains a screenshot owned by this session. `from_zoom` actions return `zoom_context_missing` when the zoom was never created or was replaced; call `get_window_state`, then `zoom`, again on the same connection.","inputSchema":{"additionalProperties":false,"properties":{"pid":{"description":"Optional target pid. When omitted, the driver resolves the unique current snapshot for this session and window.","type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"window_id":{"description":"CGWindowID from list_windows.","type":"integer"},"x1":{"description":"Left edge of region in screenshot pixels.","type":"number"},"x2":{"description":"Right edge of region in screenshot pixels.","type":"number"},"y1":{"description":"Top edge of region in screenshot pixels.","type":"number"},"y2":{"description":"Bottom edge of region in screenshot pixels.","type":"number"}},"required":["window_id","x1","y1","x2","y2"],"type":"object"},"name":"zoom","risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["session.lifecycle.start","session.capture_scope"],"description":"Optionally create or return a lifecycle session before acting. For multi-call work, prefer a short public `session` label and repeat it on every call that accepts it; an omitted value uses the authenticated transport lease's implicit session instead. This tool is optional because an ordinary action can create or reuse a named run directly. Use it to set the initial cursor theme before acting or to revive a public name after it has ended; ordinary actions never revive ended names. `capture_scope` is deprecated compatibility input; new callers select window or desktop modality per action. Idempotent.","inputSchema":{"additionalProperties":true,"properties":{"capture_scope":{"description":"Deprecated compatibility policy. New callers select window or desktop\nmodality on each action instead of storing it on the session.","enum":["auto","window","desktop"],"type":"string"},"cursor_theme":{"description":"Optional initial cursor theme. The host applies it before the cursor is\nfirst made visible, avoiding a flash of the default theme.","properties":{"reduced_motion":{"default":"auto","enum":["auto","on","off"],"type":"string"},"theme_id":{"type":"string"}},"required":["theme_id"],"type":["object","null"]},"session":{"description":"Optional stable public label for this run (e.g. \"research-run-1\").\nWhen omitted, the authenticated transport lease's implicit session is\ncreated or returned.","type":"string"}},"required":[],"type":"object"},"name":"start_session","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"active":{"type":"boolean"},"capture_scope":{"enum":["auto","window","desktop"],"type":"string"},"desktop_capture_authorized":{"type":"boolean"},"desktop_unlocked":{"type":"boolean"},"effective_scope":{"enum":["window","desktop"],"type":"string"},"escalation_detail":{"anyOf":[{"type":"string"},{"type":"null"}]},"escalation_reason":{"anyOf":[{"enum":["ax_tree_pixel_mismatch","background_delivery_failed","foreground_ineffective","no_window_target","other"],"type":"string"},{"type":"null"}]},"revived":{"type":"boolean"},"session":{"type":"string"}},"required":["session","capture_scope","effective_scope","desktop_capture_authorized","desktop_unlocked","escalation_reason","escalation_detail","active","revived"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["session.lifecycle.read"],"description":"Read content-free lifecycle, cursor, recording, and idle status for one session visible to this authenticated transport. Omit `session` to inspect its implicit session.","inputSchema":{"additionalProperties":true,"properties":{"session":{"description":"Optional public label. When omitted, inspect the caller's attached\nimplicit session.","type":"string"}},"required":[],"type":"object"},"name":"get_session","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"client_kind":{"enum":["cli","direct","mcp","python_sdk","typescript_sdk"],"type":"string"},"cursor_visible":{"type":"boolean"},"expires_in_seconds":{"minimum":0,"type":"integer"},"idle_seconds":{"minimum":0,"type":"integer"},"implicit":{"type":"boolean"},"recording_active":{"type":"boolean"},"session":{"anyOf":[{"type":"string"},{"type":"null"}]},"state":{"enum":["active","ending"],"type":"string"},"transport":{"enum":["cli","daemon","mcp_stdio","mcp_http"],"type":"string"}},"required":["session","implicit","state","client_kind","transport","cursor_visible","recording_active","idle_seconds","expires_in_seconds"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["session.lifecycle.list"],"description":"List content-free lifecycle summaries attached to this authenticated transport lease. It does not enumerate other callers' sessions.","inputSchema":{"additionalProperties":true,"properties":{"cursor":{"description":"Opaque continuation cursor returned by a previous call.","type":"string"},"limit":{"description":"Maximum number of content-free summaries to return (default 50, max\n100). Ordinary agent transports are scoped to their own lease.","minimum":0,"type":["integer","null"]}},"required":[],"type":"object"},"name":"list_sessions","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"next_cursor":{"anyOf":[{"type":"string"},{"type":"null"}]},"sessions":{"items":{"properties":{"client_kind":{"enum":["cli","direct","mcp","python_sdk","typescript_sdk"],"type":"string"},"cursor_visible":{"type":"boolean"},"expires_in_seconds":{"minimum":0,"type":"integer"},"idle_seconds":{"minimum":0,"type":"integer"},"implicit":{"type":"boolean"},"recording_active":{"type":"boolean"},"session":{"anyOf":[{"type":"string"},{"type":"null"}]},"state":{"enum":["active","ending"],"type":"string"},"transport":{"enum":["cli","daemon","mcp_stdio","mcp_http"],"type":"string"}},"required":["session","implicit","state","client_kind","transport","cursor_visible","recording_active","idle_seconds","expires_in_seconds"],"type":"object"},"type":"array"}},"required":["sessions","next_cursor"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["session.lifecycle.end"],"description":"End one visible lifecycle session and run its cursor, recording, configuration, and other cleanup hooks exactly once. Omit `session` to end the authenticated transport's implicit session. Idempotent.","inputSchema":{"additionalProperties":true,"properties":{"session":{"description":"Optional public label to end. When omitted, end the caller's attached\nimplicit session.","type":"string"}},"required":[],"type":"object"},"name":"end_session","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"active":{"const":false},"session":{"type":"string"}},"required":["session","active"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}}]}+{"driver_version":"0.31.0","patch_sha256":"45c28ca5fc737649b85ddebbb57ef837490b6349ecc1e0ac04782bfee58bdbe8","schema_version":1,"source_commit":"5272e492d61b96caf08e3bf434d91126c1f3dccc","source_tag":"cua-driver-rs-v0.31.0","source_variant":"avibe-raw-single-click-v1","tools":[{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["app.list"],"description":"List macOS apps — both currently running and installed-but-not-running — with per-app state flags:\n\n- running: is a process for this app live? (pid is 0 when false)\n- active: is it the system-frontmost app? (implies running)\n- launch_path: filesystem path to the `.app` bundle, when known. Pass this to `launch_app` to start the app cold.\n- kind: `\"desktop\"` for `.app` bundles on macOS.\n- last_used: RFC3339 timestamp from the bundle's filesystem mtime, when readable; otherwise null.\n\nStandalone running entries include only apps with NSApplicationActivationPolicyRegular — background helpers and system UI agents are filtered out. Installed apps resolve their running/pid state against all live processes by bundle identifier, so an installed app whose process runs as an accessory (LSUIElement / menu-bar apps, e.g. Cua Driver itself) still reports its live pid. Installed apps come from scanning /Applications, /Applications/Utilities, ~/Applications, /System/Applications, and /System/Applications/Utilities.\n\nUse this for \"is X installed?\" as well as \"is X running?\". For per-window state — on-screen, on-current-Space, minimized, window titles — call list_windows instead. For just opening an app — running or not — call launch_app({bundle_id: ...}) directly; list_apps is not a prerequisite.","inputSchema":{"additionalProperties":false,"properties":{"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"list_apps","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"apps":{"items":{"properties":{"active":{"type":"boolean"},"bundle_id":{"type":["string","null"]},"kind":{"type":["string","null"]},"last_used":{"type":["string","null"]},"launch_path":{"type":["string","null"]},"name":{"type":"string"},"pid":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["pid","name","running","active"],"type":"object"},"type":"array"}},"required":["apps"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["window.list"],"description":"List all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state.\n\nPer-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, space_ids, current_space_id (the active Space on that window's display), and on_current_space. The top-level current_space_id is WindowServer's main/global active Space and can differ from a record's current_space_id when displays use independent Spaces. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.","inputSchema":{"additionalProperties":false,"properties":{"on_screen_only":{"description":"When true, drop windows not on the current Space. Default false.","type":"boolean"},"pid":{"description":"Optional pid filter. When set, only this pid's windows are returned.","type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"list_windows","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"current_space_id":{"minimum":0,"type":["integer","null"]},"windows":{"items":{"properties":{"app_name":{"type":"string"},"bounds":{"properties":{"height":{"type":"number"},"width":{"type":"number"},"x":{"type":"number"},"y":{"type":"number"}},"required":["x","y","width","height"],"type":"object"},"current_space_id":{"minimum":0,"type":["integer","null"]},"is_on_screen":{"type":"boolean"},"layer":{"format":"int32","type":["integer","null"]},"minimized":{"type":["boolean","null"]},"on_current_space":{"type":["boolean","null"]},"pid":{"maximum":4294967295,"minimum":0,"type":["integer","null"]},"space_ids":{"items":{"minimum":0,"type":"integer"},"type":["array","null"]},"title":{"type":"string"},"window_id":{"minimum":0,"type":"integer"},"z_index":{"description":"Higher values are closer to the front. Null means the provider cannot observe stacking order; callers must not infer an order from array position or treat null as zero.","type":["integer","null"]}},"required":["window_id","pid","app_name","title","bounds","is_on_screen","z_index"],"type":"object"},"type":"array"}},"required":["windows"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":false,"readOnlyHint":true},"capabilities":["accessibility.window_state","accessibility.tree","accessibility.tree.structured","accessibility.tree.bounded","accessibility.element_tokens","screen.capture","screen.capture.window"],"description":"Walk a running app's AX tree and return BOTH a structured `elements` array (preferred) AND a Markdown rendering of the same tree (back-compat). Every actionable element is tagged with [element_index N] in the markdown and as `element_index` in the structured array; pass each element's `element_token` to click, type_text, press_key, etc.\n\nINVARIANT: call get_window_state once per turn per (pid, window_id) before any element action. The next snapshot of the window replaces this one, stales its element tokens, and lists the replaced ids in `invalidated_snapshot_ids`.\n\nPREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value` (the element's text/AXValue when present — use it to verify what a field holds), `actions` (names of AX actions exposed by the element, omitted when empty), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side.\n\nAlways returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces: Electron echo-confirms, Catalyst null values, virtualized off-viewport rows with `h:1` frames). You choose the modality at ACTION time, not here: an element ax action (pass `element_token` → the accessibility rung) or an element px action (pass `x`,`y` → the pixel rung, read straight off this screenshot). `capture_mode` is deprecated and ignored. Pass `include_screenshot:false` to skip the grab and get the tree only — the cheap path when you're just re-indexing before an element ax action.\n\nThe mirror image: pass `include_accessibility_tree:false` to SKIP the AX walk entirely (the expensive part, bounded by timeout_ms) and return just the screenshot plus window metadata — `window_bounds`, `screenshot_scale`, `screenshot_width`/`screenshot_height`, `app_name`, and `window_title` — the capture-only path for rendering a live window preview / picture-in-picture without paying for perception. Setting BOTH `include_accessibility_tree:false` and `include_screenshot:false` is an error (nothing to return). Optional `max_image_dimension` overrides the configured screenshot long-edge limit for this call; use 0 for native resolution. The legacy `max_dimension` remains a tighter cap for compatibility.\n\nThe snapshot is SCOPED to `window_id`: a window_id that no longer exists is refused with `window_id_not_found`, and one owned by another process is refused with `window_owner_pid_mismatch` naming the real `owner_pid` to retry with (macOS hosts a sandboxed app's Open/Save panel out-of-process, so its window belongs to the panel service, not the app). If the window is live under this pid but its accessibility surface can't be resolved, the tree comes back EMPTY with `degraded_reason: ax_window_unresolved` and the screenshot of the requested window; background input is refused until it resolves, so re-snapshot or act with `delivery_mode:\"foreground\"`. When that pid is an app still launching (its window exists before it answers accessibility), the walk first waits up to `timeout_ms` for it; if it never answers, the tree comes back EMPTY with `degraded_reason: ax_app_launching`, `truncated: true` and `truncation_reason: app_lookup_timeout`. A window on another Space still resolves by its exact CGWindowID. This tool never returns another surface's elements under your window_id. Before exposing a screenshot, its raw dimensions are validated as a coherent 1x/2x representation of the requested WindowServer bounds. `px_frame_mismatch` or `px_capture_unavailable` omits an unprovable screenshot/pixel frame instead of guessing a transform; the truthful AX payload remains available.\n\nOptional `query` projects both tree_markdown and structured `elements` to matching lines plus their ancestor chain (case-insensitive substring). The element_index values are unchanged, the complete snapshot remains actionable, and `element_count` continues to report its total size; `filtered_element_count` reports the projected response size.\n\nOptional `max_elements` / `max_depth` bound the AX walk to mitigate context-window blow-up on Electron / Obsidian / large web apps that produce 10k+ element trees. When applied, BOTH the markdown and the structured elements are truncated identically. Omit both for current default behaviour (≤2 000 elements, depth ≤25).","inputSchema":{"additionalProperties":false,"properties":{"capture_mode":{"description":"DEPRECATED and ignored. get_window_state always returns BOTH the element tree and a screenshot — ground on both. The modality is chosen at action time by how you address the target: an element ax action (element_token) or an element px action (x,y). Any value (including the old \"som\"/\"screenshot\" aliases) is accepted but has no effect.","enum":["ax","vision"],"type":"string"},"include_accessibility_tree":{"description":"Default true — walk the AX tree and return `elements` + `tree_markdown` alongside the screenshot. Set false to SKIP the AX walk entirely (the expensive part, bounded by timeout_ms) and return just the screenshot plus window metadata (bounds, scale, app_name, window_title) — the capture-only path for rendering a live window preview / picture-in-picture. Mirrors include_screenshot. Setting BOTH include_accessibility_tree:false AND include_screenshot:false is an error (nothing to return).","type":"boolean"},"include_screenshot":{"description":"Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return the tree only (the cheap path when you're just re-indexing before an element ax action; saves the image tokens + screen-grab latency). screenshot_out_file still forces a capture to disk.","type":"boolean"},"max_depth":{"description":"Cap on the AX-tree walk depth. Nodes whose rendered indent would exceed this are omitted. Omit for the default (25). Lower this for deep menu/Electron trees.","minimum":1,"type":"integer"},"max_dimension":{"description":"Optional cap on the returned screenshot's long edge, in pixels (aspect ratio preserved) — the cheap path for a small preview / thumbnail. Applied on top of the session/global max_image_dimension ceiling; the tighter of the two wins. Omit for the configured default.","minimum":1,"type":"integer"},"max_elements":{"description":"Cap on the total number of AX nodes walked. Truncates depth-first; markdown and structured elements truncate together. Omit for the default (2 000). Lower this for Electron / Obsidian / large web apps that produce 10k+ element trees and blow context windows.","minimum":1,"type":"integer"},"max_image_dimension":{"description":"Per-call override for the returned screenshot's long edge in pixels. An explicit value wins over the session/global setting; 0 returns native resolution. Omit to preserve configured behavior.","minimum":0,"type":"integer"},"pid":{"description":"Target process ID.","type":"integer"},"query":{"description":"Case-insensitive filter for tree_markdown and structured elements. Returns matching actionable rows plus their actionable ancestors without renumbering element_index values.","type":"string"},"screenshot_out_file":{"description":"When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output will contain screenshot_file_path instead.","type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"timeout_ms":{"default":1000,"description":"Wall-clock budget in milliseconds for the accessibility-tree walk (default 1000, min 100, max 120000). Bounds the WHOLE walk. When the budget runs out the tool returns the PARTIAL tree it has, flagged with `truncated: true`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `elements_complete: false`; retry with a larger value (e.g. 5000) or narrow with `query` / `max_depth`.","maximum":120000,"minimum":100,"type":"integer"},"window_id":{"description":"Target window ID from list_windows.","type":"integer"}},"required":["pid","window_id"],"type":"object"},"name":"get_window_state","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"app_name":{"type":["string","null"]},"degraded":{"type":["boolean","null"]},"degraded_reason":{"type":["string","null"]},"element_count":{"minimum":0,"type":["integer","null"]},"elements":{"items":{"properties":{"actions":{"items":{"type":"string"},"type":["array","null"]},"depth":{"minimum":0,"type":"integer"},"element_index":{"minimum":0,"type":"integer"},"element_token":{"type":["string","null"]},"enabled":{"type":["boolean","null"]},"frame":{"properties":{"h":{"type":"number"},"w":{"type":"number"},"x":{"type":"number"},"y":{"type":"number"}},"required":["x","y","w","h"],"type":["object","null"]},"in_web_content":{"type":["boolean","null"]},"label":{"type":["string","null"]},"max":{"type":["number","null"]},"min":{"type":["number","null"]},"parent_index":{"minimum":0,"type":["integer","null"]},"role":{"type":"string"},"selected":{"type":["boolean","null"]},"value":{"type":["string","null"]},"value_description":{"type":["string","null"]}},"required":["element_index","role","depth"],"type":"object"},"type":["array","null"]},"elements_complete":{"type":["boolean","null"]},"filtered_element_count":{"minimum":0,"type":["integer","null"]},"pid":{"minimum":0,"type":"integer"},"returned_element_count":{"minimum":0,"type":["integer","null"]},"screenshot_file_path":{"type":["string","null"]},"screenshot_frame_valid":{"type":["boolean","null"]},"screenshot_height":{"minimum":0,"type":["integer","null"]},"screenshot_mime_type":{"type":["string","null"]},"screenshot_scale":{"type":["number","null"]},"screenshot_width":{"minimum":0,"type":["integer","null"]},"snapshot_id":{"type":["string","null"]},"total_element_count":{"minimum":0,"type":["integer","null"]},"tree_markdown":{"type":["string","null"]},"truncated":{"type":["boolean","null"]},"truncation_reason":{"type":["string","null"]},"window_bounds":{"properties":{"height":{"type":"number"},"width":{"type":"number"},"x":{"type":"number"},"y":{"type":"number"}},"required":["x","y","width","height"],"type":["object","null"]},"window_id":{"minimum":0,"type":"integer"},"window_title":{"type":["string","null"]}},"required":["pid","window_id"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r3","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":false,"readOnlyHint":true},"capabilities":["state.verify","state.verify.window","state.verify.element","state.observe.visual_evidence"],"description":"Deterministically verify bounded predicates against one exact window. The driver evaluates structured window/accessibility state and may return the final screenshot as uninterpreted visual evidence for a multimodal caller. Predicate results are satisfied, unsatisfied, or unknown; unknown never implies success. Accessibility projections are conservative: absence remains unknown unless the observed search domain is proven exhaustive.","inputSchema":{"additionalProperties":false,"properties":{"expect":{"description":"One to eight predicates, combined with logical AND.","items":{"additionalProperties":false,"properties":{"element":{"additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"exists":{"description":"Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.","type":"boolean"},"selected":{"type":["boolean","null"]},"selector":{"additionalProperties":false,"properties":{"label_contains":{"minLength":1,"type":"string"},"role":{"minLength":1,"type":"string"}},"required":[],"type":"object"},"value_equals":{"type":["string","null"]}},"required":["selector"],"type":["object","null"]},"window":{"additionalProperties":false,"properties":{"bounds":{"additionalProperties":false,"properties":{"height":{"type":"number"},"tolerance_px":{"maximum":100,"minimum":0,"type":"number"},"width":{"type":"number"},"x":{"type":"number"},"y":{"type":"number"}},"required":["x","y","width","height"],"type":["object","null"]},"exists":{"type":["boolean","null"]}},"type":["object","null"]}},"required":[],"type":"object"},"maxItems":8,"minItems":1,"type":"array"},"include_screenshot":{"description":"Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.","type":["boolean","null"]},"pid":{"description":"Exact process whose window may be observed.","minimum":1,"type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session. This\nfield never selects capture modality or authorization.","type":"string"},"stable_samples":{"default":2,"description":"Consecutive satisfied samples required before returning success.","maximum":5,"minimum":1,"type":"integer"},"timeout_ms":{"default":5000,"description":"Bounded wait. Zero performs one sample.","maximum":10000,"minimum":0,"type":"integer"},"window_id":{"description":"Exact native window identifier.","type":"integer"}},"required":["pid","window_id","expect"],"type":"object"},"name":"verify_state","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"elapsed_ms":{"minimum":0,"type":"integer"},"predicates":{"items":{"properties":{"index":{"minimum":0,"type":"integer"},"observed_json":{"anyOf":[{"type":"string"},{"type":"null"}]},"status":{"enum":["satisfied","unsatisfied","unknown"],"type":"string"},"unknown_reason":{"anyOf":[{"enum":["invalid_predicate","unsupported_predicate","untrusted_source","multi_match","target_missing","observation_unavailable","stability_unproven"],"type":"string"},{"type":"null"}]}},"required":["index","status","unknown_reason","observed_json"],"type":"object"},"type":"array"},"samples":{"minimum":0,"type":"integer"},"stable":{"type":"boolean"},"status":{"enum":["satisfied","unsatisfied","unknown"],"type":"string"}},"required":["status","stable","elapsed_ms","samples","predicates"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":true,"readOnlyHint":false},"capabilities":["app.launch"],"description":"Launch a macOS app in the background — the target does NOT come to the foreground.\n\nProvide either `bundle_id` (preferred — unambiguous, e.g. `com.apple.calculator`) or `name` (e.g. \"Calculator\"). If both are given, bundle_id wins.\n\nOptional `urls` are handed to the app as open targets — for Finder, pass a folder path to open a backgrounded Finder window there.\n\nBrowser DevTools setup belongs to `browser_prepare`, which can prove that a separate isolated profile is driver-owned before enabling CDP.\n\nOptional `webkit_inspector_port`: opens a WebKit inspector server on the specified port (sets WEBKIT_INSPECTOR_SERVER=127.0.0.1:N + TAURI_WEBVIEW_AUTOMATION=1). Use this for Tauri/WebKit-based apps.\n\nOptional `creates_new_application_instance`: when true, forces a new app instance even if one is already running (passes -n to open). Reach for this when another agent or session may drive the SAME app concurrently — it returns a fresh pid + window so each session acts on its own isolated window instead of clobbering one shared instance. Without it, single-instance apps (Calculator, many utilities) hand every caller the same window, so two sessions fight over it.\n\nOptional `additional_arguments`: extra argv strings appended after --args.\n\nReturns the launched app's pid, bundle_id, name, and a `windows` array (same shape as `list_windows`) so callers can skip an extra round-trip before `get_window_state(pid, window_id)`. `launch_state` distinguishes whether the request was sent, the process is running, and a window is ready. When the focus-steal belt-and-braces demotion check ran (target pid ≠ prior frontmost), the response also includes `self_activation_suppressed: bool` — true if focus stayed with the prior frontmost, false if the launched app held focus despite the re-demote attempt.","inputSchema":{"additionalProperties":false,"properties":{"additional_arguments":{"description":"Extra arguments appended after --args when launching.","items":{"type":"string"},"type":"array"},"bundle_id":{"description":"App bundle identifier, e.g. com.apple.calculator. Preferred over name.","type":"string"},"creates_new_application_instance":{"description":"When true, force a new app instance even if already running (open -n). Use for concurrent multi-agent/multi-session work so each session gets an isolated instance + window instead of sharing one — on single-instance apps (e.g. Calculator) every caller otherwise gets the same window and the sessions clobber each other.","type":"boolean"},"name":{"description":"App display name. Used only when bundle_id is absent.","type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"urls":{"description":"Optional file paths or URLs to open with the app (e.g. a folder path for Finder).","items":{"type":"string"},"type":"array"},"webkit_inspector_port":{"description":"Open a WebKit inspector server on this port (sets WEBKIT_INSPECTOR_SERVER env var).","type":"integer"}},"type":"object"},"name":"launch_app","risk":{"class":"r1","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["app.kill"],"description":"Force-terminate a process by pid (kill -9 equivalent on macOS / Linux; taskkill /F equivalent on Windows). Use as escalation when the cooperative close path (hotkey cmd+q on macOS, click-the-X on Windows) failed to make the process exit. Unsaved state is lost — prefer the cooperative path first.","inputSchema":{"additionalProperties":false,"properties":{"pid":{"description":"PID of the process to terminate.","type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"required":["pid"],"type":"object"},"name":"kill_app","risk":{"class":"r3","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["window.frame.set"],"description":"Set one exact top-level window's frame in the desktop-coordinate space reported by list_windows and verify the resulting geometry through an independent readback.","inputSchema":{"additionalProperties":false,"properties":{"height":{"minimum":1,"type":"number"},"pid":{"minimum":1,"type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"width":{"minimum":1,"type":"number"},"window_id":{"minimum":1,"type":"integer"},"x":{"type":"number"},"y":{"type":"number"}},"required":["pid","window_id","x","y","width","height"],"type":"object"},"name":"set_window_frame","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.click","input.pointer.click.left","accessibility.element_tokens","input.delivery_mode"],"description":"Click against a target pid. **Prefer `element_token` over pixel coordinates** — the token works on backgrounded / minimized / hidden / off-Space windows, identifies one exact snapshot element, and tells you what you're clicking via the cached element's role + label. Reach for `x, y` only when the target is a canvas / video / WebGL / custom-drawn surface that doesn't appear in the AX tree.\n\nTwo addressing modes:\n\n- element_token (from get_window_state): AX action path. Works on backgrounded/hidden windows. No cursor move, no focus steal. The snapshot cache is scoped per (pid, window_id) and is replaced by the next snapshot of the same window — re-snapshot every turn before clicking.\n\n- x, y (window-local screenshot pixels, top-left origin of the PNG returned by get_window_state): CGEvent path. Synthesizes mouse events and posts to pid. Use modifier for cmd/shift/option/ctrl. Needs a visible on-screen window to anchor the conversion.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat — omit it and you get the legacy left-click behaviour. Pixel path: routes through the CGEvent left/right/middle mouse-button primitives. AX path: \"right\" maps to AXShowMenu (same surface as the dedicated `right_click` tool); \"middle\" has no AX equivalent and falls back to a pixel middle-click at the element's center. For a window-addressed background pixel left-click, click_mode:\"raw\" bypasses the AX hit-test optimization and posts exactly one routed mouse down/up pair. Keep the default click_mode:\"auto\" unless AXPress was observed to return a hollow success.\naction: press (default), show_menu, pick, confirm, cancel, open.\nfrom_zoom: set true after a zoom call to auto-translate zoom-image pixel coordinates to full-window space.","inputSchema":{"additionalProperties":false,"properties":{"action":{"description":"AX action: press, show_menu, pick, confirm, cancel, open.","type":"string"},"button":{"description":"Mouse button. Default: \"left\" — omit for legacy left-click behaviour. Pixel path uses the matching CGEvent primitive; AX path maps \"right\" to AXShowMenu and falls back to a pixel middle-click at the element's center for \"middle\".","enum":["left","right","middle"],"type":"string"},"capture_id":{"description":"Optional immutable source capture ID returned by get_window_state or get_desktop_state. With x,y, Driver atomically admits and consumes that exact capture before dispatch; stale, mismatched, or out-of-bounds captures are refused without fallback.","type":"string"},"click_mode":{"description":"Pixel single-click delivery selector. Default \"auto\" preserves the AX-first background optimization. \"raw\" is opt-in and valid only for an exact pid + window_id background left-click with x/y, count=1, and no modifiers; it bypasses AX hit-testing and uses the existing routed CGEvent path.","enum":["auto","raw"],"type":"string"},"count":{"description":"Click count (pixel path only). Default 1.","type":"integer"},"debug_image_out":{"description":"Optional file path. When set on a pixel-addressed click, captures a fresh screenshot, draws a red crosshair at (x, y), and writes the PNG. Use to verify coordinate spaces. Requires window_id; incompatible with from_zoom.","type":"string"},"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": perform the AX action or post the CGEvent without fronting. \"foreground\": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. Modified clicks require \"foreground\" so macOS observes physical modifier-key state. A generic click has no independent postcondition read-back, except selection of list-like AX rows whose AXSelected state can be confirmed; otherwise confirm the effect from a fresh state snapshot. Use the agent loop: background AX (element_token) → snapshot → background pixel (x/y) → snapshot → delivery_mode:\"foreground\".","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"from_zoom":{"description":"When true, x and y are in the last zoom image for this pid; driver translates back to full-window coordinates.","type":"boolean"},"modifier":{"description":"Modifier keys: cmd, shift, option/alt, ctrl.","items":{"type":"string"},"type":"array"},"pid":{"description":"Target process ID.","type":"integer"},"scope":{"description":"Coordinate frame for a windowless screen-absolute click (default \"window\"). Pass \"desktop\" when sending x,y with NO pid/window_id — the coordinates are then true screen pixels (read from get_desktop_state with scope=\"desktop\"). Per-call; not a setting.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},"window_id":{"description":"Target window ID. Omit when element_token is supplied (the token carries it).","type":"integer"},"x":{"description":"X in screenshot pixels. A window target uses the get_window_state PNG; a desktop target uses the native get_desktop_state PNG. The driver reverses Retina backing scale and any window-image downscale.","type":"number"},"y":{"description":"Y in screenshot pixels from the image selected by target.","type":"number"}},"required":[],"type":"object"},"name":"click","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.click","input.pointer.click.left","input.pointer.click.double","accessibility.element_tokens","input.delivery_mode"],"description":"Double-click at (x, y) or on an AX element identified by element_token.\n\nAX path (element_token provided): performs `AXOpen` when the element advertises it (Finder items, openable list rows/cells); otherwise resolves the element's on-screen center and falls back to a pixel double-click there.\n\nPixel path (x, y provided): two down/up pairs ~80 ms apart at the given coordinates.","inputSchema":{"additionalProperties":false,"properties":{"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"pid":{"type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"window_id":{"description":"CGWindowID. Omit when element_token is supplied (the token carries it).","type":"integer"},"x":{"description":"Screen X coordinate (pixel path).","type":"number"},"y":{"description":"Screen Y coordinate (pixel path).","type":"number"}},"required":["pid"],"type":"object"},"name":"double_click","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.click","input.pointer.click.right","accessibility.element_tokens","input.delivery_mode"],"description":"Right-click against a target pid. Two addressing modes:\n\n- `element_token` (from the last `get_window_state` snapshot) — performs `AXShowMenu` on the cached element. Pure AX RPC, works on backgrounded / hidden windows, no cursor move or focus steal. Requires a prior `get_window_state(pid, window_id)` in this turn.\n\n- `x`, `y` — synthesizes `rightMouseDown` / `rightMouseUp` CGEvent pair posted to the pid. Driver converts image-pixel → screen-point internally. `modifier` forces the CGEvent path (AX actions don't propagate modifier keys).\n\nExactly one of `element_token` or (`x` AND `y`) must be provided. `pid` always required.","inputSchema":{"additionalProperties":false,"properties":{"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"modifier":{"description":"Modifier keys held during the right-click: cmd/shift/option/ctrl. Pixel path only.","items":{"type":"string"},"type":"array"},"pid":{"description":"Target process ID.","type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"window_id":{"description":"CGWindowID. Omit when element_token is supplied (the token carries it).","type":"integer"},"x":{"description":"X in window-local screenshot pixels. Must be provided together with y.","type":"number"},"y":{"description":"Y in window-local screenshot pixels. Must be provided together with x.","type":"number"}},"required":["pid"],"type":"object"},"name":"right_click","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.drag","input.delivery_mode"],"description":"Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels — the same space get_window_state returns. Top-left origin of the target's window.\n\nUse for: marquee/lasso selection, drag-and-drop, resizing via a handle, scrubbing a slider, repositioning a panel.\n\n`duration_ms` (default 500) is the wall-clock budget for the path between mouse-down and mouse-up; `steps` (default 20) is the number of intermediate mouseDragged events linearly interpolated along the path. Increase both for slower, more human drags; decrease for snap gestures.\n\n`modifier` keys (cmd/shift/option/ctrl) are held across the entire gesture.\n\nWhen `from_zoom` is true, coordinates are in the last zoom image for this pid; the driver maps them back to window coordinates before dispatching.","inputSchema":{"additionalProperties":false,"properties":{"button":{"description":"Mouse button used for the drag. Default: left.","enum":["left","right","middle"],"type":"string"},"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"duration_ms":{"description":"Wall-clock duration of the drag path between mouseDown and mouseUp. Default: 500.","maximum":10000,"minimum":0,"type":"integer"},"from_x":{"description":"Drag-start X in window-local screenshot pixels. Top-left origin.","type":"number"},"from_y":{"description":"Drag-start Y in window-local screenshot pixels. Top-left origin.","type":"number"},"from_zoom":{"description":"When true, coordinates are in the last zoom image for this pid; driver maps back to window coordinates.","type":"boolean"},"modifier":{"description":"Modifier keys held across the entire gesture: cmd/shift/option/ctrl.","items":{"type":"string"},"type":"array"},"pid":{"description":"Target process ID.","type":"integer"},"scope":{"default":"window","description":"Use desktop with no pid/window_id for native get_desktop_state screenshot coordinates.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"steps":{"description":"Number of intermediate mouseDragged events linearly interpolated along the path. Default: 20.","maximum":200,"minimum":1,"type":"integer"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"to_x":{"description":"Drag-end X in window-local screenshot pixels.","type":"number"},"to_y":{"description":"Drag-end Y in window-local screenshot pixels.","type":"number"},"window_id":{"description":"CGWindowID for the window the pixel coordinates were measured against. Optional only when pid owns exactly one eligible top-level window; otherwise the action refuses with ambiguous_window_target.","type":"integer"}},"required":["from_x","from_y","to_x","to_y"],"type":"object"},"name":"drag","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.keyboard.type","input.keyboard.type.terminal_safe","accessibility.element_tokens","input.delivery_mode"],"description":"Insert text into the target pid via `AXSetAttribute(kAXSelectedText)`. Works for standard Cocoa text fields and text views. No keystrokes are synthesized — special keys (Return / Escape / arrows) go through `press_key` / `hotkey`. For Chromium / Electron inputs that don't implement `kAXSelectedText`, the tool falls back to CGEvent character synthesis automatically when the estimated route stays within the daemon transport budget. Longer synthesized routes are refused before character events and return a safe chunk size; one-call AX insertion remains uncapped.\n\nOptional `element_token` (from the last `get_window_state` snapshot) directs the write to a specific field. Without `element_token`, the write goes to the pid's currently focused element.\n\nWEB CONTENT (Chromium/WebKit/Electron — browser tabs, Slack, VS Code, X's compose box): AXValue is not independent proof that the renderer/DOM observed an AX write or synthesized keystrokes. The driver detects this at the element level (an AXWebArea ancestor) and refuses to trust AXValue-only read-back there. Electron AX targets that are web content or cannot be proven native refuse background delivery before mutation because the AX route cannot establish exact renderer focus; use the px form or explicit foreground delivery. Other web-content paths return effect:\"unverifiable\" + escalation, never a false \"confirmed\" (a browser's own native address bar/toolbar stays trusted). For a browser TAB the reliable path is the `page` tool (drives the DOM via CDP); for an embedded web view use this tool's px form: pass x,y (no element_token) to pixel-click the field then type, in one call. NOTE: a px focus-click won't reliably open+focus a CLOSED control; AX-press to open/activate it first (works in the background), then px-type. Always confirm via the screenshot; if px-background still drops, escalate to delivery_mode:\"foreground\".","inputSchema":{"additionalProperties":false,"properties":{"delay_ms":{"description":"Milliseconds between characters in the CGEvent fallback path. Default 30. Ignored when the AX path succeeds.","maximum":200,"minimum":0,"type":"integer"},"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": AX insert, then CGEvent keystrokes if needed — no focus steal; native controls can be confirmed via AXValue read-back, while web-content writes remain effect:\"unverifiable\". \"foreground\": briefly front the window, type, restore the prior frontmost — the explicit last resort for focus-sensitive surfaces (e.g. WhatsApp/Catalyst) where background keystrokes don't land. Re-call with \"foreground\" when a background attempt remains unverifiable and a fresh snapshot shows the text did not appear.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"pid":{"description":"Target process ID.","type":"integer"},"scope":{"default":"window","description":"Use desktop with no pid/window_id to type into the frontmost application.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"text":{"description":"Text to insert at the target's cursor.","type":"string"},"window_id":{"description":"CGWindowID. Omit when element_token is supplied (the token carries it).","type":"integer"},"x":{"description":"Screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the AX path can't reach. Read straight off the get_window_state PNG, same convention as click.","type":"number"},"y":{"description":"Screenshot-pixel Y of the field (see x).","type":"number"}},"required":["text"],"type":"object"},"name":"type_text","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.keyboard.press","accessibility.element_tokens","input.delivery_mode"],"description":"Press and release a single key. Follows the same `delivery_mode` ladder as click/type_text — it does NOT raise the window by default:\n• `background` (default): post to the pid WITHOUT fronting/raising — the auth-message path (Chromium-safe). With element_token it focuses that AX element first. `window_id` only targets; it does not raise.\n• `foreground`: guard and briefly front the exact window, focus an addressed AX element when supplied, send a genuine HID key transition so Chromium content, inline editors, and native menu equivalents receive it, then restore prior frontmost. Requires window_id.\n\nA key press is confirmed only when a bounded native AX value/selection read-back changes on the same control. Otherwise a successfully attempted post remains effect:\"unverifiable\" without implying delivery failure or recommending foreground. Key names: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, plus any letter or digit. Modifiers array: cmd, shift, option/alt, ctrl, fn.","inputSchema":{"additionalProperties":false,"properties":{"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"key":{"description":"Key name: return, tab, escape, up, down, etc.","type":"string"},"modifiers":{"description":"Modifier keys: cmd, shift, option/alt, ctrl, fn.","items":{"type":"string"},"type":"array"},"pid":{"type":"integer"},"scope":{"default":"window","description":"Use desktop with no pid/window_id to send the key to the frontmost application.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"window_id":{"description":"Target window. Required for delivery_mode:\"foreground\". Does NOT itself raise the window — raising is gated on delivery_mode.","type":"integer"},"x":{"description":"Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the AX path can't focus. Pass with y, no element_token.","type":"number"},"y":{"description":"Screenshot-pixel Y (see x).","type":"number"}},"required":["key"],"type":"object"},"name":"press_key","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.keyboard.hotkey","input.delivery_mode"],"description":"Press a key combination — e.g. `[\"cmd\", \"c\"]` for Copy, `[\"cmd\", \"shift\", \"4\"]` for screenshot selection. Follows the same `delivery_mode` ladder as click/type_text — it does NOT raise the window by default:\n• `background` (default): post the combo to the target pid WITHOUT fronting or raising it — uses the macOS 14+ auth-message envelope so Chromium/Electron accept it as trusted live input. With an AX target, focus that exact element first. No top-level focus steal. `window_id` here only targets the combo; it does not raise.\n• `foreground`: briefly front the window (NSMenu path, < 1 ms via SLPSSetFrontProcessWithOptions) so native menu key-equivalents (Cmd+Z, Cmd+W) dispatch, then restore the prior frontmost — the explicit escalation for menu-bar shortcuts on non-Chromium apps that ignore a background combo. With an AX target or x,y, the focused field receives the chord through the foreground HID queue (needed by native Chromium fields such as the omnibox). Requires window_id.\n\nA combo is never driver-verifiable (no read-back) → effect:\"unverifiable\"; confirm via screenshot. NOTE: a keyboard combo does NOT focus a text field — to type into a backgrounded Electron input, establish real renderer focus with a PIXEL click first, then `type_text`. If an app only accepts paste, call `clipboard_write`, then `clipboard_read` and verify its types (and text when applicable) before selecting or replacing editor content; only then send Cmd+V.\n\nRecognized modifiers: cmd/command, shift, option/alt, ctrl/control, fn. Non-modifier keys use the same vocabulary as `press_key`. Order: modifiers first, one non-modifier last.","inputSchema":{"additionalProperties":false,"properties":{"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"keys":{"description":"Modifier(s) and one non-modifier key, e.g. [\"cmd\", \"c\"].","items":{"type":"string"},"minItems":2,"type":"array"},"pid":{"description":"Target process ID.","type":"integer"},"scope":{"default":"window","description":"Use desktop with no pid/window_id to send the chord to the frontmost application.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"window_id":{"description":"Target window. Required for delivery_mode:\"foreground\" (the NSMenu activation needs a window). Does NOT itself raise the window — raising is gated on delivery_mode.","type":"integer"},"x":{"description":"Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the combo (so e.g. Cmd+V pastes into that field). Pass with y. Use for Chromium/Electron surfaces the background combo can't reach.","type":"number"},"y":{"description":"Screenshot-pixel Y (see x).","type":"number"}},"required":["keys"],"type":"object"},"name":"hotkey","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":true,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.keyboard.type","accessibility.element_tokens"],"description":"Set a value on a UI element. Two modes depending on element role:\n\n- **AXPopUpButton / select dropdown**: finds the child option whose title or value matches `value` (case-insensitive) and AXPresses it directly — the native macOS popup menu is never opened, so focus is never stolen. Use this for HTML <select> elements in Safari or any native NSPopUpButton.\n\n- **All other elements**: writes AXValue directly (sliders, steppers, date pickers, native text fields that expose settable AXValue).\n\nFor free-form text entry into web inputs, prefer `type_text_chars` which synthesises key events — AXValue writes are ignored by WebKit.","inputSchema":{"additionalProperties":false,"properties":{"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"pid":{"type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"value":{"description":"New value. AX will coerce to the element's native type.","type":"string"},"window_id":{"description":"CGWindowID. Omit when element_token is supplied (the token carries it).","type":"integer"}},"required":["pid","value"],"type":"object"},"name":"set_value","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":true,"readOnlyHint":false},"capabilities":["input.pointer.scroll","accessibility.element_tokens","input.delivery_mode"],"description":"Scroll the target pid. Two paths, picked by how you address the scroll:\n\n• **Targeted wheel path** — when you pass a target, either `element_token` (preferred) or window-local `x, y` pixels: the driver synthesizes a real mouse-wheel event (CGEventCreateScrollWheelEvent, at that screen point. The renderer hit-tests the wheel at the cursor, so the scroll lands on whatever element is under the point — exactly like physically rolling the wheel over it. This is the ONLY way to scroll a nested `overflow:auto` region (e.g. a scrollable <div> with no tabindex): such regions never take keyboard focus, so the keystroke path below no-ops on them. Use this for inner/nested scrollers in web views.\n\n• **Keystroke path (focused region)** — when you pass NO target (just pid + direction): synthesizes PageDown/PageUp (by='page') or Down/Up arrows (by='line'); horizontal uses Left/Right arrows. Drives the focused / page scroller only.\n\nMapping: by='page' → larger step; by='line' → smaller step; amount = number of wheel notches (targeted path) or keystroke repetitions (keystroke path).","inputSchema":{"additionalProperties":false,"properties":{"amount":{"description":"Pixel-wheel path: number of wheel notches. Keystroke path: number of keystroke repetitions. Larger requests are clamped to the maximum. Default: 3.","maximum":50,"minimum":1,"type":"integer"},"by":{"description":"Scroll granularity. Default: line.","enum":["line","page"],"type":"string"},"delivery_mode":{"description":"Best-effort-background ladder rung (default \"background\"). \"background\": inject without fronting or raising the target — no focus steal. \"foreground\": briefly front the target, act, then restore the prior frontmost — the explicit last resort when a background attempt didn't land. Re-call with \"foreground\" only for the action that needs it.","enum":["background","foreground"],"type":"string"},"direction":{"description":"Scroll direction.","enum":["up","down","left","right"],"type":"string"},"element_token":{"description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.","type":"string"},"pid":{"type":"integer"},"scope":{"default":"window","description":"Use desktop with x,y and no pid/window_id for native get_desktop_state screenshot coordinates.","enum":["window","desktop"],"type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"target":{"anyOf":[{"description":"Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.","oneOf":[{"additionalProperties":true,"properties":{"kind":{"const":"window","type":"string"},"pid":{"minimum":0,"type":"integer"},"window_id":{"minimum":0,"type":"integer"}},"required":["kind","pid","window_id"],"type":"object"},{"additionalProperties":true,"properties":{"display_id":{"type":"string"},"kind":{"const":"desktop","type":"string"}},"required":["kind","display_id"],"type":"object"}]},{"type":"null"}]},"window_id":{"type":"integer"},"x":{"description":"Window-local screenshot X (top-left origin of the PNG from get_window_state). With `y`, routes through the pixel-wheel path at this point — use for a scrollable surface that isn't in the AX tree. Requires window_id to anchor the window→screen conversion.","type":"number"},"y":{"description":"Window-local screenshot Y. See `x`.","type":"number"}},"required":["direction"],"type":"object"},"name":"scroll","outputSchema":{"anyOf":[{"additionalProperties":false,"properties":{"delivery":{"additionalProperties":false,"properties":{"delivered_count":{"minimum":0,"type":["integer","null"]},"mode":{"enum":["background","foreground","not_applicable","unknown"],"type":"string"}},"required":["mode"],"type":["object","null"]},"effect":{"enum":["confirmed","partial","unverifiable","suspected_noop","refused"],"type":"string"},"error":{"additionalProperties":false,"properties":{"code":{"type":"string"},"hint":{"type":["string","null"]}},"required":["code"],"type":["object","null"]},"escalation":{"additionalProperties":false,"properties":{"reason":{"enum":["route_unavailable","delivery_failed","effect_unconfirmed","suspected_noop","permission_required"],"type":"string"},"target":{"enum":["pixel","foreground","page","session"],"type":"string"}},"required":["target","reason"],"type":["object","null"]},"evidence":{"items":{"additionalProperties":false,"properties":{"detail":{"type":["string","null"]},"kind":{"enum":["value_readback","window_change"],"type":"string"}},"required":["kind"],"type":"object"},"type":["array","null"]},"route":{"enum":["accessibility","synthetic_events","global_input","system_api","dom","trusted_input"],"type":"string"},"summary":{"type":["string","null"]}},"required":["effect","route"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":false,"readOnlyHint":true},"capabilities":["clipboard.read","clipboard.types"],"description":"List available system clipboard types and optionally return privacy-sensitive plain text. Clipboard content is never retained in telemetry.","inputSchema":{"additionalProperties":false,"properties":{"include_text":{"default":false,"description":"Return plain-text clipboard content in addition to the available types.\nClipboard content is privacy-sensitive and is never retained in telemetry.","type":"boolean"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"required":[],"type":"object"},"name":"clipboard_read","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"content_redacted_from_telemetry":{"type":"boolean"},"privacy_sensitive":{"type":"boolean"},"supported":{"type":"boolean"},"text":{"anyOf":[{"type":"string"},{"type":"null"}]},"types":{"items":{"type":"string"},"type":"array"}},"required":["supported","types","text","privacy_sensitive","content_redacted_from_telemetry"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["clipboard.write","clipboard.write.text","clipboard.write.image","clipboard.write.file_url","clipboard.types"],"description":"Replace the system clipboard with exactly one value: plain text, an image from an absolute local path, or a file URL from an absolute local path. Returns the available types for read-back before paste.","inputSchema":{"additionalProperties":false,"properties":{"file_path":{"description":"Absolute path to a local file to place on the clipboard as a file URL.","type":"string"},"image_path":{"description":"Absolute path to a local image to place on the clipboard.","type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"text":{"description":"Plain text to place on the clipboard.","type":"string"}},"required":[],"type":"object"},"name":"clipboard_write","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"content_redacted_from_telemetry":{"type":"boolean"},"privacy_sensitive":{"type":"boolean"},"supported":{"type":"boolean"},"types":{"items":{"type":"string"},"type":"array"},"written_type":{"type":"string"}},"required":["supported","written_type","types","privacy_sensitive","content_redacted_from_telemetry"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"active","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["screen.dimensions"],"description":"Return the logical size of the main display in points plus its backing scale factor. Agents click in points; Retina displays have scale_factor 2.0. Requires no TCC permissions.","inputSchema":{"additionalProperties":false,"properties":{"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"get_screen_size","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"height":{"type":"number"},"scale_factor":{"type":"number"},"width":{"type":"number"}},"required":["width","height","scale_factor"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":false,"openWorldHint":false,"readOnlyHint":true},"capabilities":["screen.capture","screen.dimensions"],"description":"Capture the full display in true screen pixels, full size unless `max_image_dimension` caps it. Use its PNG as the coordinate source for actions whose target is {kind:\"desktop\",display_id:\"primary\"}. Returns the true screen size and backing scale factor. Vision-only: no AX tree walk.","inputSchema":{"additionalProperties":false,"properties":{"max_image_dimension":{"description":"Optional long-edge cap for the returned PNG, in pixels (aspect ratio preserved). Omitted or 0 returns the full-size capture. When the cap downsizes the image, the response reports `screenshot_original_width/height`, and x/y read off it for this session's later scope:\"desktop\" actions (or passed with its `capture_id`) are mapped back to the full-size frame automatically.","minimum":0,"type":"integer"},"screenshot_out_file":{"description":"Write PNG here instead of base64.","type":"string"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"get_desktop_state","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"agent_overlay_capture":{"additionalProperties":false,"properties":{"method":{"type":["string","null"]},"reason":{"type":["string","null"]},"status":{"oneOf":[{"const":"excluded","type":"string"},{"const":"not_present","type":"string"},{"const":"not_excluded","type":"string"}]}},"required":["status"],"type":["object","null"]},"display":{"type":"string"},"platform":{"enum":["macos","linux","windows"],"type":"string"},"scale_factor":{"type":"number"},"screen_height":{"type":"integer"},"screen_width":{"type":"integer"},"screenshot_file_path":{"type":"string"},"screenshot_height":{"type":"integer"},"screenshot_mime_type":{"const":"image/png"},"screenshot_width":{"type":"integer"}},"required":["platform","display","screenshot_width","screenshot_height","screen_width","screen_height","scale_factor","screenshot_mime_type"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r3","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["screen.cursor.position"],"description":"Return the current mouse cursor position in screen points (origin top-left).","inputSchema":{"additionalProperties":false,"properties":{"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"get_cursor_position","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"available":{"type":"boolean"},"source":{"type":"string"},"x":{"type":"number"},"y":{"type":"number"}},"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":[],"description":"Single-call end-to-end driver diagnostics. Designed to let downstream consumers ship one stable call instead of stitching together check_permissions, doctor, version, bundle attribution, and platform capability status. On macOS, prompt-capable direct capture is deliberately skipped; use `cua-driver permissions grant` to verify it explicitly. cua-driver owns the health model; consumers stay thin.\n\nInput — all optional:\n {\n \"include\": [\"<check_name>\", ...], // run only these\n \"skip\": [\"<check_name>\", ...] // skip these\n }\nIf both are given, `include` wins.\n\nCanonical check names:\n macOS : binary_version, platform_supported, session_active,\n bundle_identity, tcc_accessibility, tcc_screen_recording,\n ax_capability, screen_capture_capability\n Windows: binary_version, platform_supported, session_active,\n ax_capability (via UIA), screen_capture_capability (via DXGI)\n Linux : binary_version, platform_supported, session_active,\n ax_capability (via AT-SPI), screen_capture_capability (via X11)\n\nOutput — stable contract, schema_version=\"1\":\n {\n \"schema_version\": \"1\",\n \"platform\": \"darwin\" | \"win32\" | \"linux\",\n \"driver_version\": \"<semver>\",\n \"overall\": \"ok\" | \"degraded\" | \"failed\",\n \"checks\": [\n {\n \"name\": \"<one of the canonical names above>\",\n \"status\": \"pass\" | \"fail\" | \"skip\",\n \"message\": \"<one-line summary, always present>\",\n \"hint\": \"<remediation step, present when status=fail>\",\n \"data\": { /* check-specific structured fields */ }\n },\n ...\n ]\n }\n\n`overall` rules:\n - `ok` — every non-skipped check passes\n - `degraded` — at least one non-core check fails (binary is still usable)\n - `failed` — any core check fails (binary_version, platform_supported, session_active)\n\nStability: schema_version=\"1\" is the contract. Future breaking changes will be `\"2\"`. Adding new check names under the same schema_version is non-breaking; consumers must tolerate unknown check names.","inputSchema":{"additionalProperties":false,"properties":{"include":{"description":"Only run these checks (canonical names). Wins over `skip`.","items":{"type":"string"},"type":"array"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"skip":{"description":"Skip these checks (canonical names). Ignored when `include` is set.","items":{"type":"string"},"type":"array"}},"type":"object"},"name":"health_report","risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["accessibility.tree","accessibility.tree.structured"],"description":"Return a lightweight snapshot of the desktop: running regular apps and on-screen visible windows with their bounds, z-order, and owner pid.\n\nFor the full AX subtree of a single window (with interactive element indices you can click by), use `get_window_state` instead — that's the heavy per-window tool. This one is a fast discovery read that needs no TCC grants.","inputSchema":{"additionalProperties":false,"properties":{"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"}},"type":"object"},"name":"get_accessibility_tree","risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["screen.capture","screen.capture.window","screen.capture.region"],"description":"Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in screenshot pixel coordinates, with 20% padding added on each side. The output image is at most 500 px wide.\n\nAfter a zoom, pass `from_zoom=true` to click/type_text to auto-translate coordinates back to full-window space. Coordinate actions return `screenshot_context_missing` when no current snapshot contains a screenshot owned by this session. `from_zoom` actions return `zoom_context_missing` when the zoom was never created or was replaced; call `get_window_state`, then `zoom`, again on the same connection.","inputSchema":{"additionalProperties":false,"properties":{"pid":{"description":"Optional target pid. When omitted, the driver resolves the unique current snapshot for this session and window.","type":"integer"},"session":{"description":"For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.","type":"string"},"window_id":{"description":"CGWindowID from list_windows.","type":"integer"},"x1":{"description":"Left edge of region in screenshot pixels.","type":"number"},"x2":{"description":"Right edge of region in screenshot pixels.","type":"number"},"y1":{"description":"Top edge of region in screenshot pixels.","type":"number"},"y2":{"description":"Bottom edge of region in screenshot pixels.","type":"number"}},"required":["window_id","x1","y1","x2","y2"],"type":"object"},"name":"zoom","risk":{"class":"r2","enforcement":"active","operation_sensitive":true,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["session.lifecycle.start","session.capture_scope"],"description":"Optionally create or return a lifecycle session before acting. For multi-call work, prefer a short public `session` label and repeat it on every call that accepts it; an omitted value uses the authenticated transport lease's implicit session instead. This tool is optional because an ordinary action can create or reuse a named run directly. Use it to set the initial cursor theme before acting or to revive a public name after it has ended; ordinary actions never revive ended names. `capture_scope` is deprecated compatibility input; new callers select window or desktop modality per action. Idempotent.","inputSchema":{"additionalProperties":true,"properties":{"capture_scope":{"description":"Deprecated compatibility policy. New callers select window or desktop\nmodality on each action instead of storing it on the session.","enum":["auto","window","desktop"],"type":"string"},"cursor_theme":{"description":"Optional initial cursor theme. The host applies it before the cursor is\nfirst made visible, avoiding a flash of the default theme.","properties":{"reduced_motion":{"default":"auto","enum":["auto","on","off"],"type":"string"},"theme_id":{"type":"string"}},"required":["theme_id"],"type":["object","null"]},"session":{"description":"Optional stable public label for this run (e.g. \"research-run-1\").\nWhen omitted, the authenticated transport lease's implicit session is\ncreated or returned.","type":"string"}},"required":[],"type":"object"},"name":"start_session","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"active":{"type":"boolean"},"capture_scope":{"enum":["auto","window","desktop"],"type":"string"},"desktop_capture_authorized":{"type":"boolean"},"desktop_unlocked":{"type":"boolean"},"effective_scope":{"enum":["window","desktop"],"type":"string"},"escalation_detail":{"anyOf":[{"type":"string"},{"type":"null"}]},"escalation_reason":{"anyOf":[{"enum":["ax_tree_pixel_mismatch","background_delivery_failed","foreground_ineffective","no_window_target","other"],"type":"string"},{"type":"null"}]},"revived":{"type":"boolean"},"session":{"type":"string"}},"required":["session","capture_scope","effective_scope","desktop_capture_authorized","desktop_unlocked","escalation_reason","escalation_detail","active","revived"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["session.lifecycle.read"],"description":"Read content-free lifecycle, cursor, recording, and idle status for one session visible to this authenticated transport. Omit `session` to inspect its implicit session.","inputSchema":{"additionalProperties":true,"properties":{"session":{"description":"Optional public label. When omitted, inspect the caller's attached\nimplicit session.","type":"string"}},"required":[],"type":"object"},"name":"get_session","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"client_kind":{"enum":["cli","direct","mcp","python_sdk","typescript_sdk"],"type":"string"},"cursor_visible":{"type":"boolean"},"expires_in_seconds":{"minimum":0,"type":"integer"},"idle_seconds":{"minimum":0,"type":"integer"},"implicit":{"type":"boolean"},"recording_active":{"type":"boolean"},"session":{"anyOf":[{"type":"string"},{"type":"null"}]},"state":{"enum":["active","ending"],"type":"string"},"transport":{"enum":["cli","daemon","mcp_stdio","mcp_http"],"type":"string"}},"required":["session","implicit","state","client_kind","transport","cursor_visible","recording_active","idle_seconds","expires_in_seconds"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":false,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":true},"capabilities":["session.lifecycle.list"],"description":"List content-free lifecycle summaries attached to this authenticated transport lease. It does not enumerate other callers' sessions.","inputSchema":{"additionalProperties":true,"properties":{"cursor":{"description":"Opaque continuation cursor returned by a previous call.","type":"string"},"limit":{"description":"Maximum number of content-free summaries to return (default 50, max\n100). Ordinary agent transports are scoped to their own lease.","minimum":0,"type":["integer","null"]}},"required":[],"type":"object"},"name":"list_sessions","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"next_cursor":{"anyOf":[{"type":"string"},{"type":"null"}]},"sessions":{"items":{"properties":{"client_kind":{"enum":["cli","direct","mcp","python_sdk","typescript_sdk"],"type":"string"},"cursor_visible":{"type":"boolean"},"expires_in_seconds":{"minimum":0,"type":"integer"},"idle_seconds":{"minimum":0,"type":"integer"},"implicit":{"type":"boolean"},"recording_active":{"type":"boolean"},"session":{"anyOf":[{"type":"string"},{"type":"null"}]},"state":{"enum":["active","ending"],"type":"string"},"transport":{"enum":["cli","daemon","mcp_stdio","mcp_http"],"type":"string"}},"required":["session","implicit","state","client_kind","transport","cursor_visible","recording_active","idle_seconds","expires_in_seconds"],"type":"object"},"type":"array"}},"required":["sessions","next_cursor"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r0","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}},{"annotations":{"destructiveHint":true,"idempotentHint":true,"openWorldHint":false,"readOnlyHint":false},"capabilities":["session.lifecycle.end"],"description":"End one visible lifecycle session and run its cursor, recording, configuration, and other cleanup hooks exactly once. Omit `session` to end the authenticated transport's implicit session. Idempotent.","inputSchema":{"additionalProperties":true,"properties":{"session":{"description":"Optional public label to end. When omitted, end the caller's attached\nimplicit session.","type":"string"}},"required":[],"type":"object"},"name":"end_session","outputSchema":{"anyOf":[{"additionalProperties":true,"properties":{"active":{"const":false},"session":{"type":"string"}},"required":["session","active"],"type":"object"},{"additionalProperties":true,"anyOf":[{"required":["refusal"]},{"required":["status"]},{"required":["code"]}],"type":"object"}],"type":"object"},"risk":{"class":"r1","enforcement":"metadata_only","operation_sensitive":false,"version":"1"}}]}diff --git a/desktop/runtime-host/src/computer_use.rs b/desktop/runtime-host/src/computer_use.rsindex b9ebebec6..ab122b776 100644--- a/desktop/runtime-host/src/computer_use.rs+++ b/desktop/runtime-host/src/computer_use.rs@@ -18,7 +18,7 @@ pub const COMPUTER_USE_STATE_FILE: &str = "computer-use.json";pub const COMPUTER_USE_LOCK_FILE: &str = "computer-use.lock";pub const COMPUTER_USE_SOCKET_FILE: &str = "cua-driver.sock";pub const COMPUTER_USE_DRIVER_VERSION: &str = "0.31.0";-pub const COMPUTER_USE_TOOL_SNAPSHOT_SHA256: &str = "b03c3e48d1b00c8fe7c0e8b9813eb5ea104ad38313672fc4b68af203a827f43b";+pub const COMPUTER_USE_TOOL_SNAPSHOT_SHA256: &str = "4257e598f5a6ad7e632b48e96c03257d7c5ed618a93b72820b2d4fef0f196acf";pub const RUNTIME_TOO_OLD_REASON: &str = "runtime_too_old";pub const RUNTIME_UNAVAILABLE_REASON: &str = "runtime_unavailable";pub const FAILURE_LIMIT: usize = 3;@@ -687,6 +687,24 @@ mod tests {);}+ #[test]+ fn tool_snapshot_digest_matches_the_runtime_pin() {+ use sha2::{Digest, Sha256};++ const SNAPSHOT: &[u8] =+ include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../cua-driver/tools-v0.31.0.json"));+ let actual = format!("{:x}", Sha256::digest(SNAPSHOT));+ assert_eq!(actual, COMPUTER_USE_TOOL_SNAPSHOT_SHA256);++ let mut mutated = SNAPSHOT.to_vec();+ mutated[0] ^= 1;+ assert_ne!(+ format!("{:x}", Sha256::digest(mutated)),+ COMPUTER_USE_TOOL_SNAPSHOT_SHA256,+ "a changed snapshot must fail the runtime pin"+ );+ }+#[test]fn shell_lock_reports_a_same_user_startup_race_without_replacing_the_holder() {let directory = std::env::temp_dir().join(format!(diff --git a/desktop/scripts/prepare-cua-driver.py b/desktop/scripts/prepare-cua-driver.pyindex cd10366c5..5ecb523d0 100644--- a/desktop/scripts/prepare-cua-driver.py+++ b/desktop/scripts/prepare-cua-driver.py@@ -1,5 +1,5 @@#!/usr/bin/env python3-"""Verify and prepare the pinned macOS Cua Driver nested helper."""+"""Build the pinned and explicitly patched macOS Cua Driver helper."""from __future__ import annotations@@ -8,6 +8,7 @@ import hashlibimport jsonimport osfrom pathlib import Path+from pathlib import PurePosixPathimport reimport shutilimport subprocess@@ -47,6 +48,7 @@ def load_manifest(path: Path = SOURCES) -> dict:or driver.get("version") != "0.31.0"or driver.get("tag") != "cua-driver-rs-v0.31.0"or driver.get("source_commit") != "5272e492d61b96caf08e3bf434d91126c1f3dccc"+ or SHA256.fullmatch(str(driver.get("tool_snapshot_sha256"))) is Noneor driver.get("license") != "MIT"or not isinstance(driver.get("release_base_url"), str)or driver.get("release_checksums_asset") != "checksums.txt"@@ -54,10 +56,42 @@ def load_manifest(path: Path = SOURCES) -> dict:or not isinstance(macos, dict)or macos.get("asset") != "cua-driver-rs-0.31.0-darwin-universal-binary.tar.gz"or SHA256.fullmatch(str(macos.get("sha256"))) is None+ or not isinstance(driver.get("source"), dict)+ or driver["source"].get("archive")+ != "cua-5272e492d61b96caf08e3bf434d91126c1f3dccc.tar.gz"+ or driver["source"].get("url")+ != "https://codeload.github.com/trycua/cua/tar.gz/5272e492d61b96caf08e3bf434d91126c1f3dccc"+ or SHA256.fullmatch(str(driver["source"].get("sha256"))) is None+ or driver["source"].get("root")+ != "cua-5272e492d61b96caf08e3bf434d91126c1f3dccc"+ or not isinstance(driver.get("patch"), dict)+ or driver["patch"].get("variant") != "avibe-raw-single-click-v1"+ or driver["patch"].get("file")+ != "patches/0001-macos-raw-single-click.patch"+ or SHA256.fullmatch(str(driver["patch"].get("sha256"))) is None+ or driver["patch"].get("contract") != "click.click_mode=raw"+ or not isinstance(driver["patch"].get("upstream_refs"), list)+ or not driver["patch"]["upstream_refs"]+ or not isinstance(driver["patch"].get("upgrade_plan"), str)+ or not driver["patch"]["upgrade_plan"].strip()or payload.get("managed_policy") != "policy.yaml"or payload.get("tool_snapshot") != "tools-v0.31.0.json"):raise ValueError("invalid pinned Cua Driver source manifest")+ patch = path.parent / driver["patch"]["file"]+ if not patch.is_file() or digest(patch) != driver["patch"]["sha256"]:+ raise ValueError("pinned Cua Driver patch hash mismatch")+ try:+ snapshot = json.loads(TOOL_SNAPSHOT.read_text(encoding="utf-8"))+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:+ raise ValueError("pinned Cua Driver tool snapshot is unreadable") from exc+ if (+ not isinstance(snapshot, dict)+ or snapshot.get("patch_sha256") != driver["patch"]["sha256"]+ ):+ raise ValueError("pinned Cua Driver tool snapshot patch hash mismatch")+ if digest(TOOL_SNAPSHOT) != driver["tool_snapshot_sha256"]:+ raise ValueError("pinned Cua Driver tool snapshot hash mismatch")if "perception" in json.dumps(payload).lower():raise ValueError("Cua perception assets must not enter the Phase 1 package")verify_managed_policy_surface()@@ -107,115 +141,156 @@ def download(url: str, target: Path) -> None:shutil.copyfileobj(response, output)-def checksums(text: str) -> dict[str, str]:- parsed: dict[str, str] = {}- for line in text.splitlines():- match = re.fullmatch(r"([0-9a-f]{64}) ([A-Za-z0-9._+-]+)", line)- if match:- parsed[match.group(2)] = match.group(1)- return parsed+def extract_driver_source(manifest: dict, archive_path: Path, destination: Path) -> Path:+ """Extract only the pinned driver subtree from the verified source archive."""+ source = manifest["driver"]["source"]+ if digest(archive_path) != source["sha256"]:+ raise ValueError("Cua Driver source archive hash mismatch")+ archive_root = PurePosixPath(source["root"])+ driver_root = archive_root / "libs" / "cua-driver"+ extracted_files = 0+ with tarfile.open(archive_path, "r:gz") as archive:+ for member in archive.getmembers():+ path = PurePosixPath(member.name)+ if path.is_absolute() or ".." in path.parts:+ raise ValueError("unsafe Cua Driver source archive path")+ if path.parts[: len(driver_root.parts)] != driver_root.parts:+ continue+ relative = PurePosixPath(*path.parts[1:])+ target = destination.joinpath(*relative.parts)+ if member.isdir():+ target.mkdir(parents=True, exist_ok=True)+ continue+ if not member.isfile():+ raise ValueError("Cua Driver source subtree contains a non-regular file")+ extracted = archive.extractfile(member)+ if extracted is None:+ raise ValueError("Cua Driver source archive member could not be read")+ target.parent.mkdir(parents=True, exist_ok=True)+ with target.open("wb") as output:+ shutil.copyfileobj(extracted, output)+ target.chmod(member.mode & 0o777)+ extracted_files += 1+ source_root = destination+ cargo_manifest = source_root / "libs" / "cua-driver" / "rust" / "Cargo.toml"+ if extracted_files == 0 or not cargo_manifest.is_file():+ raise ValueError("Cua Driver source archive is missing the pinned Rust workspace")+ return source_root-def verify_release_inputs(manifest: dict, archive: Path, checksum_file: Path) -> None:- driver = manifest["driver"]- asset = driver["macos"]["asset"]- if digest(checksum_file) != driver["release_checksums_sha256"]:- raise ValueError("Cua Driver checksums.txt hash mismatch")- release_hashes = checksums(checksum_file.read_text(encoding="utf-8"))- expected = driver["macos"]["sha256"]- if release_hashes.get(asset) != expected:- raise ValueError("Cua Driver release checksums do not contain the pinned asset hash")- if digest(archive) != expected:- raise ValueError("Cua Driver archive hash mismatch")---def safe_member(archive: tarfile.TarFile, name: str) -> tarfile.TarInfo:- matches = [- member- for member in archive.getmembers()- if member.isfile() and Path(member.name).name == name- ]- if len(matches) != 1:- raise ValueError(f"Cua Driver archive must contain exactly one {name}")- member = matches[0]- if member.name.startswith("/") or ".." in Path(member.name).parts:- raise ValueError("unsafe Cua Driver archive path")- return member+def run(command: list[str], *, merge_stderr: bool = True) -> str:+ return subprocess.check_output(+ command,+ text=True,+ stderr=subprocess.STDOUT if merge_stderr else None,+ )-def run(command: list[str]) -> str:- return subprocess.check_output(command, text=True, stderr=subprocess.STDOUT)++def apply_driver_patch(manifest: dict, source_root: Path) -> Path:+ patch = ROOT / "cua-driver" / manifest["driver"]["patch"]["file"]+ subprocess.run(+ ["git", "-C", str(source_root), "apply", "--check", str(patch)],+ check=True,+ )+ subprocess.run(+ ["git", "-C", str(source_root), "apply", str(patch)],+ check=True,+ )+ return patch+++def build_driver(manifest: dict, source_root: Path, target: str) -> Path:+ workspace = source_root / "libs" / "cua-driver" / "rust"+ target_dir = ROOT / "target" / "cua-driver-patched"+ environment = {+ **os.environ,+ "CARGO_INCREMENTAL": "0",+ "CARGO_TARGET_DIR": str(target_dir),+ }+ subprocess.run(+ [+ "cargo",+ "build",+ "--locked",+ "--release",+ "-p",+ "cua-driver",+ "--bin",+ "cua-driver",+ "--target",+ target,+ ],+ cwd=workspace,+ env=environment,+ check=True,+ )+ built = target_dir / target / "release" / "cua-driver"+ if not built.is_file():+ raise ValueError("patched Cua Driver build did not produce the expected binary")+ info = run(["lipo", "-info", str(built)])+ arch = TARGET_ARCH[target]+ if arch not in info or "Non-fat file" not in info:+ raise ValueError(f"patched Cua Driver has the wrong architecture: {info}")+ return built+++def verify_built_contract(binary: Path) -> None:+ if run([str(binary), "--version"]).strip() != "cua-driver 0.31.0":+ raise ValueError("patched Cua Driver reports an unexpected version")+ try:+ docs = json.loads(+ run([str(binary), "dump-docs", "--type", "mcp"], merge_stderr=False)+ )+ click = next(tool for tool in docs["tools"] if tool.get("name") == "click")+ click_mode = click["input_schema"]["properties"]["click_mode"]+ except (json.JSONDecodeError, KeyError, StopIteration, TypeError) as exc:+ raise ValueError("patched Cua Driver does not expose click.click_mode") from exc+ if click_mode.get("enum") != ["auto", "raw"]:+ raise ValueError("patched Cua Driver click.click_mode contract is unexpected")def prepare(target: str,*,- archive: Path | None = None,- checksum_file: Path | None = None,+ source_archive: Path | None = None,output_dir: Path | None = None,) -> Path:manifest = load_manifest()arch = TARGET_ARCH[target]driver = manifest["driver"]- base_url = driver["release_base_url"].rstrip("/")destination = (output_dir or ROOT / "src-tauri" / "binaries").resolve()destination.mkdir(parents=True, exist_ok=True)output = destination / f"cua-driver-{target}"with tempfile.TemporaryDirectory(prefix="avibe-cua-driver-") as temporary:work = Path(temporary)- resolved_checksums = checksum_file or work / "checksums.txt"- resolved_archive = archive or work / driver["macos"]["asset"]- if checksum_file is None:- download(- f"{base_url}/{driver['release_checksums_asset']}",- resolved_checksums,- )- if archive is None:- download(f"{base_url}/{driver['macos']['asset']}", resolved_archive)- verify_release_inputs(manifest, resolved_archive, resolved_checksums)-- universal = work / "cua-driver"- with tarfile.open(resolved_archive, "r:gz") as source:- member = safe_member(source, "cua-driver")- extracted = source.extractfile(member)- if extracted is None:- raise ValueError("Cua Driver archive member could not be read")- with universal.open("wb") as stream:- shutil.copyfileobj(extracted, stream)- universal.chmod(0o755)- universal_sha256 = digest(universal)-- prepared = work / f"cua-driver-{arch}"- subprocess.run(- ["lipo", str(universal), "-thin", arch, "-output", str(prepared)],- check=True,- )- info = run(["lipo", "-info", str(prepared)])- if arch not in info or "Non-fat file" not in info:- raise ValueError(f"prepared Cua Driver has the wrong architecture: {info}")- subprocess.run(- ["codesign", "--verify", "--strict", "--verbose=2", str(prepared)],- check=True,- )- prepared.chmod(0o755)+ resolved_archive = source_archive or work / driver["source"]["archive"]+ if source_archive is None:+ download(driver["source"]["url"], resolved_archive)+ source_root = extract_driver_source(manifest, resolved_archive, work / "source")+ patch = apply_driver_patch(manifest, source_root)+ prepared = build_driver(manifest, source_root, target)+ verify_built_contract(prepared)temporary_output = destination / f".{output.name}.{os.getpid()}.tmp"shutil.copyfile(prepared, temporary_output)temporary_output.chmod(0o755)os.replace(temporary_output, output)provenance = {- "schema_version": 1,+ "schema_version": 2,"version": driver["version"],"tag": driver["tag"],"source_commit": driver["source_commit"],"target": target,"arch": arch,- "release_checksums_sha256": driver["release_checksums_sha256"],- "archive": driver["macos"]["asset"],- "archive_sha256": driver["macos"]["sha256"],- "extracted_universal_sha256": universal_sha256,- "thinned_upstream_sha256": digest(output),- "thinned_signature": "upstream_preserved",+ "source_archive": driver["source"]["archive"],+ "source_archive_sha256": driver["source"]["sha256"],+ "patch_variant": driver["patch"]["variant"],+ "patch_file": driver["patch"]["file"],+ "patch_sha256": digest(patch),+ "tool_snapshot_sha256": digest(TOOL_SNAPSHOT),+ "prepared_binary_sha256": digest(output),+ "build_origin": "repository_patch_built",}provenance_output = output.with_name(f"{output.name}.provenance.json")temporary_provenance = destination / f".{provenance_output.name}.{os.getpid()}.tmp"@@ -231,8 +306,7 @@ def main() -> None:parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--check-manifest", action="store_true")parser.add_argument("--target", choices=TARGET_ARCH)- parser.add_argument("--archive", type=Path)- parser.add_argument("--checksums", type=Path)+ parser.add_argument("--source-archive", type=Path)parser.add_argument("--output-dir", type=Path)args = parser.parse_args()if args.check_manifest:@@ -243,8 +317,7 @@ def main() -> None:parser.error("--target is required unless only --check-manifest is used")output = prepare(args.target,- archive=args.archive,- checksum_file=args.checksums,+ source_archive=args.source_archive,output_dir=args.output_dir,)print(output)diff --git a/docs/plans/desktop-computer-use.md b/docs/plans/desktop-computer-use.mdindex e7862c07a..78ede42a0 100644--- a/docs/plans/desktop-computer-use.md+++ b/docs/plans/desktop-computer-use.md@@ -10,6 +10,78 @@- Companion docs: `desktop-tray-lifecycle.md` (native toggle pattern),`desktop-auto-update.md` (signing), `desktop-product-gaps.md`+## Route A: raw single-click selector (2026-10-10)++Status: implementation lane, based on remote master+`0438a99916e82c9fe0387321dd65fd87b4412587`, branch+`fix/desktop-cua-raw-single-click-20261010`.++The bounded research probe showed that the existing routed window-pointer+CGEvent path can deliver a hit-tested click to a test-owned AppKit target in+the background without changing the frontmost PID or hardware cursor+coordinates.+The failure in self-drawn apps was the click tool's AX-first optimization:+`AXPress` returned success while the target did not receive `mouseDown`.++Upstream investigation found no stable, maintainer-accepted selector contract+that bypasses this hollow AX success while preserving the current default+behavior. The pinned source is `cua-driver-rs-v0.31.0` at commit+`5272e492d61b96caf08e3bf434d91126c1f3dccc`, with source archive SHA-256+`8134337f2beb1854918d8bbc5aea22d7ae00de9838ce6426f275d7ba6734d191`.+Relevant upstream work and the upgrade plan are recorded in+[`desktop/cua-driver/PATCHES.md`](../../desktop/cua-driver/PATCHES.md) and+`desktop/cua-driver/sources.json`.++The smallest accepted contract is `click_mode`:++- missing or `"auto"` keeps today's AX-first `count=1` click behavior;+- `"raw"` is opt-in and requires an exact `pid` + `window_id`, pixel `x/y`,+ background left-click delivery, `action` `press` or `click`, `count=1`, and+ no modifiers, element token, or portable target;+- raw delivery defaults to background when `delivery_mode` is omitted; when+ supplied, the field must be exactly `"background"` rather than null,+ foreground, or an unknown future value;+- raw mode bypasses only the macOS click tool's AX hit-test/`AXPress` shortcut+ and uses the existing routed CGEvent path;+- unknown values and invalid combinations are structured+ `invalid_arguments`.++Avibe admits and forwards `click_mode` only for the `click` tool, after the+existing observe-first and exact-window admission. A request observed before+the target window is admitted is never forwarded. The tool snapshot hash is+updated in the core and Runtime host, and the package workflow now builds the+patched source and emits schema-2 provenance containing source, patch, snapshot,+prepared-binary, packaged-binary, and cdhash evidence.++The package verifier keeps two generated schema surfaces distinct. The+driver's `dump-docs --type mcp` output uses `input_schema`, while the managed+MCP snapshot uses `inputSchema`. Packaging validates the former against the+built binary; `core/computer_server.py` consumes the latter. Owning tests reject+the snapshot shape at the `dump-docs` boundary so a guessed key cannot silently+replace the producer's actual contract again.++Driver provenance also keeps the unsigned and signed binary surfaces distinct.+Immediately before Tauri packaging, the workflow hashes the prepared sidecar+at `desktop/src-tauri/binaries/cua-driver-<target>` and requires it to match+`prepared_binary_sha256`. Release recording separately hashes the signed helper+inside the app as `packaged_sha256`; signing may make the two hashes differ.+Owning mutation tests reject both an arbitrary valid-looking digest and the+signed helper's digest when either is substituted for the prepared sidecar+digest.++The test-owned AppKit fixture makes `AXPress` a hollow success and counts+native `mouseDown`/`mouseUp`. Its ignored live harness is compiled in this+lane but is not run here because live GUI/TCC actions are outside the current+authorization. The owning contract tests cover snapshot/manifest locking,+source-subtree extraction, rejection of non-regular archive members, Avibe+admission/forwarding, and release provenance verification.++Route B, a bounded foreground mode for applications that reject both+accessibility actions and background synthetic events, is explicitly deferred.+It must be designed and accepted from new real evidence; it cannot become a+silent fallback for Route A. QQ/WeChat verification likewise requires a later+owner handoff after the patched artifact exists.+## BackgroundAvibe agents can run shell commands and take a desktop screenshot@@ -104,15 +176,16 @@ running. The tray keeps the shell alive after the window closes.standard nested-code location. Launch it directly with `Command`, neverthrough LaunchServices.Pin the version and SHA-256 in a sources manifest beside- `runtime-sources.json`. Fetch only the plain driver binary, never the- `cua-perception` extension, which is AGPL. Add the upstream MIT notice.- Desktop packages are per architecture (`aarch64-apple-darwin`,- `x86_64-apple-darwin`, `x86_64-pc-windows-msvc`), so ship one thin slice per- package: `lipo -thin` of the universal binary keeps Cua's valid per-slice- signature. Record the release archive, extracted universal, and thinned- upstream digests before packaging, separately from the packaged digest and- cdhash after signing. Newly produced desktop source metadata uses schema 2:- every macOS record must include this complete driver provenance block.+ `runtime-sources.json`. Fetch only the plain driver source, never the+ `cua-perception` extension, which is AGPL. Apply only the explicitly+ recorded patch variant, add the upstream MIT notice, and record the source+ archive and patch hashes. Desktop packages are per architecture+ (`aarch64-apple-darwin`, `x86_64-apple-darwin`,+ `x86_64-pc-windows-msvc`), so build one target-specific macOS helper per+ package. Record the prepared binary digest before packaging, separately from+ the packaged digest and cdhash after signing. Newly produced desktop source+ metadata uses schema 2: every macOS record must include this complete driver+ provenance block.Schema-1 metadata remains accepted only as a legacy published input. Signnested code explicitly before the outer app; `--deep` is verification-only.Windows ships `cua-driver.exe`; whether the default-offdiff --git a/scripts/desktop_release.py b/scripts/desktop_release.pyindex ea322704f..07e5a95da 100644--- a/scripts/desktop_release.py+++ b/scripts/desktop_release.py@@ -136,6 +136,12 @@ def pinned_driver_source() -> dict:"release_checksums_sha256": source["release_checksums_sha256"],"archive": source["macos"]["asset"],"archive_sha256": source["macos"]["sha256"],+ "source_archive": source["source"]["archive"],+ "source_archive_sha256": source["source"]["sha256"],+ "patch_variant": source["patch"]["variant"],+ "patch_file": source["patch"]["file"],+ "patch_sha256": source["patch"]["sha256"],+ "tool_snapshot_sha256": source["tool_snapshot_sha256"],}@@ -155,40 +161,64 @@ def validate_manifest(manifest: dict, target: str, tag: str) -> None:def driver_provenance(target: str,- driver: Path,+ prepared_driver: Path,provenance_path: Path,- packaged_cdhash: str,+ *,+ packaged_driver: Path | None = None,+ packaged_cdhash: str | None = None,) -> dict:if TARGETS[target][0] != "macos":raise ValueError("Cua Driver provenance is macOS-only")- if re.fullmatch(r"[0-9a-f]{40,64}", packaged_cdhash) is None:- raise ValueError("Invalid packaged Cua Driver cdhash")+ if prepared_driver.is_symlink() or not prepared_driver.is_file():+ raise ValueError("Invalid prepared Cua Driver provenance")provenance = json.loads(provenance_path.read_text(encoding="utf-8"))required_hashes = (- "release_checksums_sha256",- "archive_sha256",- "extracted_universal_sha256",- "thinned_upstream_sha256",+ "source_archive_sha256",+ "patch_sha256",+ "tool_snapshot_sha256",+ "prepared_binary_sha256",)pinned = pinned_driver_source()+ patched_fields = (+ "version",+ "tag",+ "source_commit",+ "source_archive",+ "source_archive_sha256",+ "patch_variant",+ "patch_file",+ "patch_sha256",+ "tool_snapshot_sha256",+ )if (- provenance.get("schema_version") != 1- or any(provenance.get(field) != value for field, value in pinned.items())+ provenance.get("schema_version") != 2+ or any(provenance.get(field) != pinned[field] for field in patched_fields)or provenance.get("target") != targetor provenance.get("arch") != TARGETS[target][1].replace("aarch64", "arm64")- or provenance.get("thinned_signature") != "upstream_preserved"+ or provenance.get("build_origin") != "repository_patch_built"or any(re.fullmatch(r"[0-9a-f]{64}", str(provenance.get(field))) is None for field in required_hashes)+ or digest(prepared_driver) != provenance.get("prepared_binary_sha256")):raise ValueError("Invalid prepared Cua Driver provenance")+ if packaged_driver is None and packaged_cdhash is None:+ return provenance+ if (+ packaged_driver is None+ or packaged_driver.is_symlink()+ or not packaged_driver.is_file()+ or re.fullmatch(r"[0-9a-f]{40,64}", str(packaged_cdhash)) is None+ ):+ raise ValueError("Invalid packaged Cua Driver provenance")return {**provenance,- "packaged_sha256": digest(driver),+ "packaged_sha256": digest(packaged_driver),"packaged_cdhash": packaged_cdhash,}def record(*, version: str, target: str, tag: str, source_sha: str,installer: Path, runtime: Path, output: Path, signing: str,+ prepared_driver: Path | None = None,driver: Path | None = None, driver_provenance_path: Path | None = None,driver_cdhash: str | None = None) -> None:if SHA.fullmatch(source_sha) is None:@@ -216,15 +246,16 @@ def record(*, version: str, target: str, tag: str, source_sha: str,"version": version, "target": target,"package_version": manifest["runtime_version"],}- driver_values = (driver, driver_provenance_path, driver_cdhash)+ driver_values = (prepared_driver, driver, driver_provenance_path, driver_cdhash)if TARGETS[target][0] == "macos":if any(value is None for value in driver_values):raise ValueError("macOS desktop artifacts require Cua Driver provenance")source["computer_use_driver"] = driver_provenance(target,- driver,+ prepared_driver,driver_provenance_path,- driver_cdhash,+ packaged_driver=driver,+ packaged_cdhash=driver_cdhash,)elif any(value is not None for value in driver_values):raise ValueError("Windows desktop artifacts cannot carry macOS Cua Driver provenance")@@ -272,29 +303,81 @@ def verify(directory: Path, tag: str, source_sha: str, *, updater_enabled: booldriver = source.pop("computer_use_driver", None)if TARGETS[target][0] == "macos":pinned = pinned_driver_source()- required_hashes = (+ legacy_fields = (+ "version",+ "tag",+ "source_commit",+ "release_checksums_sha256",+ "archive",+ "archive_sha256",+ )+ patched_fields = (+ "version",+ "tag",+ "source_commit",+ "source_archive",+ "source_archive_sha256",+ "patch_variant",+ "patch_file",+ "patch_sha256",+ "tool_snapshot_sha256",+ )+ legacy_hashes = ("release_checksums_sha256","archive_sha256","extracted_universal_sha256","thinned_upstream_sha256","packaged_sha256",)+ patched_hashes = (+ "source_archive_sha256",+ "patch_sha256",+ "tool_snapshot_sha256",+ "prepared_binary_sha256",+ "packaged_sha256",+ )if driver is None and schema_version == 1:# Schema-1 macOS artifacts predate the pinned Cua Driver# provenance block. Keep those already-published artifacts# verifiable. Schema 2 makes the block mandatory.pass- elif (- not isinstance(driver, dict)- or driver.get("schema_version") != 1- or any(driver.get(field) != value for field, value in pinned.items())- or driver.get("target") != target- or driver.get("arch") != TARGETS[target][1].replace("aarch64", "arm64")- or driver.get("thinned_signature") != "upstream_preserved"- or re.fullmatch(r"[0-9a-f]{40,64}", str(driver.get("packaged_cdhash"))) is None- or any(re.fullmatch(r"[0-9a-f]{64}", str(driver.get(field))) is None- for field in required_hashes)- ):+ elif not isinstance(driver, dict):+ raise ValueError("Desktop Cua Driver provenance mismatch")+ elif driver.get("schema_version") == 1:+ if (+ any(driver.get(field) != pinned[field] for field in legacy_fields)+ or driver.get("target") != target+ or driver.get("arch")+ != TARGETS[target][1].replace("aarch64", "arm64")+ or driver.get("thinned_signature") != "upstream_preserved"+ or re.fullmatch(+ r"[0-9a-f]{40,64}", str(driver.get("packaged_cdhash"))+ )+ is None+ or any(+ re.fullmatch(r"[0-9a-f]{64}", str(driver.get(field))) is None+ for field in legacy_hashes+ )+ ):+ raise ValueError("Desktop Cua Driver provenance mismatch")+ elif driver.get("schema_version") == 2:+ if (+ any(driver.get(field) != pinned[field] for field in patched_fields)+ or driver.get("target") != target+ or driver.get("arch")+ != TARGETS[target][1].replace("aarch64", "arm64")+ or driver.get("build_origin") != "repository_patch_built"+ or re.fullmatch(+ r"[0-9a-f]{40,64}", str(driver.get("packaged_cdhash"))+ )+ is None+ or any(+ re.fullmatch(r"[0-9a-f]{64}", str(driver.get(field))) is None+ for field in patched_hashes+ )+ ):+ raise ValueError("Desktop Cua Driver provenance mismatch")+ else:raise ValueError("Desktop Cua Driver provenance mismatch")elif driver is not None:raise ValueError("Windows desktop source unexpectedly carries Cua Driver provenance")@@ -361,9 +444,14 @@ def main() -> None:producer.add_argument("--runtime", type=Path, required=True)producer.add_argument("--output", type=Path, required=True)producer.add_argument("--signing", required=True)+ producer.add_argument("--prepared-driver", type=Path)producer.add_argument("--driver", type=Path)producer.add_argument("--driver-provenance-path", type=Path)producer.add_argument("--driver-cdhash")+ verifier = sub.add_parser("verify-driver-prepared")+ verifier.add_argument("--target", choices=TARGETS, required=True)+ verifier.add_argument("--driver", type=Path, required=True)+ verifier.add_argument("--provenance-path", type=Path, required=True)for command in ("verify", "check-remote"):consumer = sub.add_parser(command)consumer.add_argument("--directory", type=Path, required=True)@@ -384,6 +472,12 @@ def main() -> None:prepare(**args)elif command == "record":record(**args)+ elif command == "verify-driver-prepared":+ driver_provenance(+ args["target"],+ args["driver"],+ args["provenance_path"],+ )elif command == "verify":verify(**args)else:diff --git a/tests/test_computer_server.py b/tests/test_computer_server.pyindex 79d730503..9ee7da59a 100644--- a/tests/test_computer_server.py+++ b/tests/test_computer_server.py@@ -650,6 +650,23 @@ async def test_window_input_requires_observation_and_rejects_focus_routes()assert "observe_first" in first["content"][0]["text"]+ raw_before_observation = await server.call_tool(+ "click",+ {+ "session": "ses-a",+ "pid": 7,+ "window_id": 9,+ "x": 1,+ "y": 2,+ "click_mode": "raw",+ },+ )+ assert "observe_first" in raw_before_observation["content"][0]["text"]+ assert not any(+ params["arguments"].get("click_mode") == "raw"+ for _method, params in upstream.calls+ )+observed = await server.call_tool("get_window_state",{"session": "ses-a", "pid": 7, "window_id": 9},@@ -661,6 +678,20 @@ async def test_window_input_requires_observation_and_rejects_focus_routes()assert not clicked.get("isError")+ raw_after_observation = await server.call_tool(+ "click",+ {+ "session": "ses-a",+ "pid": 7,+ "window_id": 9,+ "x": 1,+ "y": 2,+ "click_mode": "raw",+ },+ )+ assert not raw_after_observation.get("isError")+ assert upstream.calls[-1][1]["arguments"]["click_mode"] == "raw"+foreground = await server.call_tool("click",{@@ -686,6 +717,40 @@ async def test_window_input_requires_observation_and_rejects_focus_routes(assert "focus_shortcut_forbidden" in shortcut["content"][0]["text"]+def test_raw_click_mode_is_admitted_only_for_exact_background_pixel_click() -> None:+ valid_raw_click = {+ "click_mode": "raw",+ "pid": 7,+ "window_id": 9,+ "x": 1,+ "y": 2,+ }++ _validate_input("click", valid_raw_click)+ _validate_input(+ "click",+ {**valid_raw_click, "delivery_mode": "background"},+ )+ with pytest.raises(ComputerServerError, match="only for the click tool"):+ _validate_input("get_window_state", {"click_mode": "raw"})+ with pytest.raises(ComputerServerError, match="exact pid"):+ _validate_input(+ "click",+ {+ **valid_raw_click,+ "modifier": ["cmd"],+ },+ )+ for delivery_mode in (None, "foreground", "future-mode"):+ with pytest.raises(ComputerServerError, match="background left-click"):+ _validate_input(+ "click",+ {**valid_raw_click, "delivery_mode": delivery_mode},+ )+ with pytest.raises(ComputerServerError, match="auto.*raw"):+ _validate_input("click", {"click_mode": "semantic"})++@pytest.mark.asyncioasync def test_real_lease_epoch_survives_handoff_and_rejects_stale_observation(tmp_path: Path,diff --git a/tests/test_computer_use.py b/tests/test_computer_use.pyindex 0f99b4d00..c904b3bdc 100644--- a/tests/test_computer_use.py+++ b/tests/test_computer_use.py@@ -8,6 +8,7 @@ import pytestfrom core.computer_use import (COMPUTER_USE_STATE_DIR_ENV,+ COMPUTER_USE_TOOL_SNAPSHOT_SHA256,COMPUTER_USE_TOOL_NAMES,ComputerUseConfigFingerprint,ComputerUseConfigReconciler,@@ -70,6 +71,24 @@ def test_tool_contract_has_exactly_the_approved_28_names() -> None:}.isdisjoint(COMPUTER_USE_TOOL_NAMES)+def test_tool_snapshot_digest_matches_core_and_manifest_pins() -> None:+ """Every consumer must approve the exact shipped snapshot bytes."""++ snapshot_path = Path("desktop/cua-driver/tools-v0.31.0.json")+ snapshot_bytes = snapshot_path.read_bytes()+ actual = hashlib.sha256(snapshot_bytes).hexdigest()+ manifest = json.loads(+ Path("desktop/cua-driver/sources.json").read_text(encoding="utf-8")+ )++ assert actual == COMPUTER_USE_TOOL_SNAPSHOT_SHA256+ assert actual == manifest["driver"]["tool_snapshot_sha256"]++ mutated = bytearray(snapshot_bytes)+ mutated[0] ^= 1+ assert hashlib.sha256(mutated).hexdigest() != COMPUTER_USE_TOOL_SNAPSHOT_SHA256++def test_desktop_state_path_ignores_avibe_home(tmp_path: Path) -> None:"""A shell-adopted Runtime must find D even under a different AVIBE_HOME."""diff --git a/tests/test_cua_driver_packaging.py b/tests/test_cua_driver_packaging.pyindex 314e46ef2..b5f19a01a 100644--- a/tests/test_cua_driver_packaging.py+++ b/tests/test_cua_driver_packaging.py@@ -1,7 +1,13 @@from __future__ import annotations+import hashlib+import io+import jsonfrom pathlib import Pathfrom runpy import run_path+import tarfile++import pytest_SCRIPT = run_path(@@ -13,3 +19,144 @@ def test_managed_policy_matches_the_pinned_28_tool_snapshot() -> None:"""Release preparation rejects policy drift before any driver is packaged."""_SCRIPT["verify_managed_policy_surface"]()+++def test_patched_source_manifest_hash_locks_the_divergence() -> None:+ manifest = _SCRIPT["load_manifest"]()+ driver = manifest["driver"]+ patch = Path("desktop/cua-driver") / driver["patch"]["file"]+ snapshot = json.loads(+ (Path("desktop/cua-driver") / manifest["tool_snapshot"]).read_text()+ )++ assert driver["patch"]["contract"] == "click.click_mode=raw"+ assert driver["patch"]["upstream_refs"]+ assert driver["patch"]["upgrade_plan"]+ assert _SCRIPT["digest"](patch) == driver["patch"]["sha256"]+ assert snapshot["patch_sha256"] == driver["patch"]["sha256"]+++def test_built_contract_reads_dump_docs_input_schema_without_stderr_contamination(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ calls: list[list[str]] = []++ def fake_check_output(command: list[str], **kwargs: object) -> str:+ calls.append(command)+ if command[1:] == ["--version"]:+ return "cua-driver 0.31.0\n"+ assert command[1:] == ["dump-docs", "--type", "mcp"]+ output = json.dumps(+ {+ "tools": [+ {+ "name": "click",+ "input_schema": {+ "properties": {+ "click_mode": {"enum": ["auto", "raw"]},+ }+ },+ }+ ]+ }+ )+ if kwargs.get("stderr") is _SCRIPT["subprocess"].STDOUT:+ return "Cua Driver sends content-free product telemetry.\n" + output+ return output++ monkeypatch.setattr(+ _SCRIPT["subprocess"], "check_output", fake_check_output+ )+ _SCRIPT["verify_built_contract"](tmp_path / "cua-driver")++ assert calls == [+ [str(tmp_path / "cua-driver"), "--version"],+ [str(tmp_path / "cua-driver"), "dump-docs", "--type", "mcp"],+ ]+++def test_built_contract_rejects_snapshot_input_schema_shape(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ snapshot = json.loads(+ Path("desktop/cua-driver/tools-v0.31.0.json").read_text()+ )++ def fake_check_output(command: list[str], **_kwargs: object) -> str:+ if command[1:] == ["--version"]:+ return "cua-driver 0.31.0\n"+ assert command[1:] == ["dump-docs", "--type", "mcp"]+ return json.dumps(snapshot)++ monkeypatch.setattr(+ _SCRIPT["subprocess"], "check_output", fake_check_output+ )++ with pytest.raises(+ ValueError,+ match="does not expose click.click_mode",+ ):+ _SCRIPT["verify_built_contract"](tmp_path / "cua-driver")+++def test_source_extraction_is_scoped_to_the_driver_subtree(tmp_path: Path) -> None:+ archive = tmp_path / "source.tar.gz"+ root = "cua-fixture"+ with tarfile.open(archive, "w:gz") as output:+ for name, body in (+ (f"{root}/libs/cua-driver/rust/Cargo.toml", b"[workspace]\n"),+ (f"{root}/libs/cua-driver/rust/src/lib.rs", b"pub fn fixture() {}\n"),+ (f"{root}/owner-data.txt", b"must not be extracted\n"),+ ):+ member = tarfile.TarInfo(name)+ member.size = len(body)+ member.mode = 0o644+ output.addfile(member, io.BytesIO(body))+ source_sha256 = hashlib.sha256(archive.read_bytes()).hexdigest()+ manifest = {+ "driver": {+ "source": {+ "root": root,+ "sha256": source_sha256,+ }+ }+ }++ extracted = _SCRIPT["extract_driver_source"](+ manifest,+ archive,+ tmp_path / "extracted",+ )++ assert (+ extracted / "libs/cua-driver/rust/Cargo.toml"+ ).read_text() == "[workspace]\n"+ assert not (extracted / "owner-data.txt").exists()+++def test_source_extraction_rejects_non_regular_driver_members(tmp_path: Path) -> None:+ archive = tmp_path / "source.tar.gz"+ root = "cua-fixture"+ with tarfile.open(archive, "w:gz") as output:+ cargo = tarfile.TarInfo(f"{root}/libs/cua-driver/rust/Cargo.toml")+ cargo.size = 0+ output.addfile(cargo, io.BytesIO())+ link = tarfile.TarInfo(f"{root}/libs/cua-driver/rust/escape")+ link.type = tarfile.SYMTYPE+ link.linkname = "/tmp/escape"+ output.addfile(link)+ manifest = {+ "driver": {+ "source": {+ "root": root,+ "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),+ }+ }+ }++ with pytest.raises(ValueError, match="non-regular"):+ _SCRIPT["extract_driver_source"](+ manifest,+ archive,+ tmp_path / "extracted",+ )diff --git a/tests/test_desktop_release.py b/tests/test_desktop_release.pyindex 328dcc63d..d8091954c 100644--- a/tests/test_desktop_release.py+++ b/tests/test_desktop_release.py@@ -46,6 +46,27 @@ def secret_references(value):return {name.upper() for name in canonical} | ({None} if other else set())+def driver_provenance_payload(target, prepared_binary_sha256):+ pinned_driver = release.pinned_driver_source()+ arch = release.TARGETS[target][1]+ return {+ "schema_version": 2,+ "version": pinned_driver["version"],+ "tag": pinned_driver["tag"],+ "source_commit": pinned_driver["source_commit"],+ "target": target,+ "arch": "arm64" if arch == "aarch64" else arch,+ "source_archive": pinned_driver["source_archive"],+ "source_archive_sha256": pinned_driver["source_archive_sha256"],+ "patch_variant": pinned_driver["patch_variant"],+ "patch_file": pinned_driver["patch_file"],+ "patch_sha256": pinned_driver["patch_sha256"],+ "tool_snapshot_sha256": pinned_driver["tool_snapshot_sha256"],+ "prepared_binary_sha256": prepared_binary_sha256,+ "build_origin": "repository_patch_built",+ }++def assemble_assets(root, source=SOURCE):"""Use the real Runtime archive writer and desktop producer, then merge downloads."""directory = root / "desktop-dist"@@ -71,26 +92,22 @@ def assemble_assets(root, source=SOURCE):output = work / "output"driver_args = {}if system == "macos":- pinned_driver = release.pinned_driver_source()+ prepared_driver = work / "prepared-cua-driver"+ prepared_driver.write_bytes(f"prepared driver for {target}".encode())driver = work / "cua-driver"driver.write_bytes(f"packaged driver for {target}".encode())provenance = work / "driver-provenance.json"- provenance.write_text(json.dumps({- "schema_version": 1,- "version": pinned_driver["version"],- "tag": pinned_driver["tag"],- "source_commit": pinned_driver["source_commit"],- "target": target,- "arch": "arm64" if arch == "aarch64" else arch,- "release_checksums_sha256":- pinned_driver["release_checksums_sha256"],- "archive": pinned_driver["archive"],- "archive_sha256": pinned_driver["archive_sha256"],- "extracted_universal_sha256": "3" * 64,- "thinned_upstream_sha256": "4" * 64,- "thinned_signature": "upstream_preserved",- }), encoding="utf-8")+ provenance.write_text(+ json.dumps(+ driver_provenance_payload(+ target,+ release.digest(prepared_driver),+ )+ ),+ encoding="utf-8",+ )driver_args = {+ "prepared_driver": prepared_driver,"driver": driver,"driver_provenance_path": provenance,"driver_cdhash": "5" * 40,@@ -253,10 +270,11 @@ def test_verify_rejects_schema_two_macos_metadata_without_driver(tmp_path):@pytest.mark.parametrize(("field", "value"),[- ("schema_version", 2),+ ("schema_version", 1),("arch", "wrong-arch"),- ("archive", "wrong-driver.tar.gz"),- ("archive_sha256", "f" * 64),+ ("source_archive", "wrong-source.tar.gz"),+ ("source_archive_sha256", "f" * 64),+ ("patch_sha256", "e" * 64),],)def test_verify_rejects_mismatched_driver_provenance(tmp_path, field, value):@@ -286,6 +304,61 @@ def test_verify_rejects_mismatched_driver_provenance(tmp_path, field, value):release.verify(directory, TAG, SOURCE)+def test_driver_provenance_binds_prepared_and_packaged_driver_bytes(tmp_path):+ target = "aarch64-apple-darwin"+ prepared = tmp_path / "prepared-cua-driver"+ prepared.write_bytes(b"prepared driver")+ packaged = tmp_path / "packaged-cua-driver"+ packaged.write_bytes(b"signed packaged driver")+ provenance = tmp_path / "driver-provenance.json"+ provenance.write_text(+ json.dumps(driver_provenance_payload(target, release.digest(prepared))),+ encoding="utf-8",+ )++ result = release.driver_provenance(+ target,+ prepared,+ provenance,+ packaged_driver=packaged,+ packaged_cdhash="5" * 40,+ )++ assert result["prepared_binary_sha256"] == release.digest(prepared)+ assert result["packaged_sha256"] == release.digest(packaged)+ assert result["prepared_binary_sha256"] != result["packaged_sha256"]++[email protected]("digest_source", ["wrong", "packaged"])+def test_driver_provenance_rejects_unbound_prepared_digest(+ tmp_path, digest_source+):+ target = "aarch64-apple-darwin"+ prepared = tmp_path / "prepared-cua-driver"+ prepared.write_bytes(b"prepared driver")+ packaged = tmp_path / "packaged-cua-driver"+ packaged.write_bytes(b"signed packaged driver")+ prepared_digest = (+ "f" * 64+ if digest_source == "wrong"+ else release.digest(packaged)+ )+ provenance = tmp_path / "driver-provenance.json"+ provenance.write_text(+ json.dumps(driver_provenance_payload(target, prepared_digest)),+ encoding="utf-8",+ )++ with pytest.raises(ValueError, match="Invalid prepared Cua Driver provenance"):+ release.driver_provenance(+ target,+ prepared,+ provenance,+ packaged_driver=packaged,+ packaged_cdhash="5" * 40,+ )++@pytest.mark.parametrize("target", release.TARGETS)def test_workflow_record_command_feeds_the_release_consumer(tmp_path, target):directory = assemble_assets(tmp_path)@@ -308,12 +381,20 @@ def test_workflow_record_command_feeds_the_release_consumer(tmp_path, target):installer = work / matrix["artifact_glob"].replace("*", "Avibe_fixture")installer.parent.mkdir(parents=True)shutil.copyfile(work / ("native installer" + release.TARGETS[target][2]), installer)+ if "darwin" in target:+ prepared_driver = work / f"desktop/src-tauri/binaries/cua-driver-{target}"+ prepared_driver.parent.mkdir(parents=True)+ shutil.copyfile(work / "prepared-cua-driver", prepared_driver)binaries = work / "bin"binaries.mkdir()(binaries / "python").symlink_to(sys.executable)(binaries / "git").write_text(f"#!/bin/sh\nprintf '%s\\n' '{SOURCE}'\n")(binaries / "git").chmod(0o755)- command = step("Record artifact hashes")["run"].replace("${{ matrix.artifact_glob }}", matrix["artifact_glob"])+ command = (+ step("Record artifact hashes")["run"]+ .replace("${{ matrix.artifact_glob }}", matrix["artifact_glob"])+ .replace("${{ matrix.target }}", target)+ )result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", command], cwd=work,env={**os.environ, "PATH": f"{binaries}{os.pathsep}{os.environ['PATH']}","AVIBE_DESKTOP_VERSION": VERSION, "RELEASE_TAG": TAG, "TARGET": target,@@ -378,6 +459,7 @@ def test_producer_rejects_incorrect_runtime_or_signing(tmp_path, mutation):driver_args = {}if "darwin" in target:driver_args = {+ "prepared_driver": work / "prepared-cua-driver","driver": work / "cua-driver","driver_provenance_path": work / "driver-provenance.json","driver_cdhash": "5" * 40,@@ -407,6 +489,12 @@ def test_workflow_preserves_manual_path_and_isolates_test_signing():assert steps.index(step("Prepare package version")) < steps.index(step("Build verified private Runtime"))credentials = step("Export Apple signing credentials when configured")assert credentials["if"] == "runner.os == 'macOS' && inputs.release_tag == ''"+ prepared_driver = step("Verify prepared Cua Driver provenance")+ assert prepared_driver["if"] == "runner.os == 'macOS'"+ assert "verify-driver-prepared" in prepared_driver["run"]+ assert steps.index(prepared_driver) + 1 == steps.index(+ step("Build self-contained installer")+ )signing = step("Verify macOS app signature matches the signing path")["run"]assert signing.count("--preserve-metadata=identifier,entitlements,flags,runtime"