Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Windows + Linux support for cua-driver via cua-driver-rs (Rust port) (#1511)
ddupontbb68021896b9
diff --git a/.github/workflows/cd-rust-cua-driver.yml b/.github/workflows/cd-rust-cua-driver.ymlnew file mode 100644index 0000000000..a8c0fc8b0a--- /dev/null+++ b/.github/workflows/cd-rust-cua-driver.yml@@ -0,0 +1,327 @@+name: "CD: Cua Driver RS (cross-platform)"++# Matches the artifact convention from cd-swift-cua-driver.yml:+# - macOS: ONE universal binary tarball + bare universal binary+# (lipo arm64 + x86_64 in a single job, no separate per-arch matrix)+# - Windows: cua-driver.exe bare + .zip+# - Linux: cua-driver bare + .tar.gz+#+# Bare binaries let curlable installers do+# `curl -L .../cua-driver-rs-vN-darwin-universal-binary.tar.gz | tar -xz`+# without needing to unpack a directory structure first.++on:+ push:+ tags:+ - "cua-driver-rs-v*"+ workflow_dispatch:+ inputs:+ version:+ description: "Version to release (without v prefix)"+ required: true+ default: "0.1.0"++permissions:+ contents: write++jobs:+ # ── Linux x86_64 ─────────────────────────────────────────────────────────────+ build-linux:+ name: linux-x86_64+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4+ - name: Determine version+ id: version+ shell: bash+ run: |+ if [[ "$GITHUB_REF" == refs/tags/cua-driver-rs-v* ]]; then+ VERSION="${GITHUB_REF#refs/tags/cua-driver-rs-v}"+ else+ VERSION="${{ inputs.version }}"+ fi+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"+ - uses: dtolnay/rust-toolchain@stable+ with:+ targets: x86_64-unknown-linux-gnu+ - uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ libs/cua-driver-rs/target+ key: rust-cua-driver-rs-linux-x86_64-${{ hashFiles('libs/cua-driver-rs/Cargo.lock', 'libs/cua-driver-rs/**/Cargo.toml') }}+ restore-keys: |+ rust-cua-driver-rs-linux-x86_64-+ - name: Build (release)+ working-directory: libs/cua-driver-rs+ run: cargo build -p cua-driver --release --target x86_64-unknown-linux-gnu+ - name: Package+ working-directory: libs/cua-driver-rs+ run: |+ VERSION="${{ steps.version.outputs.version }}"+ LABEL="linux-x86_64"+ STAGE="cua-driver-rs-${VERSION}-${LABEL}"+ mkdir -p "release/${STAGE}"+ cp "target/x86_64-unknown-linux-gnu/release/cua-driver" "release/${STAGE}/"+ cp ../../LICENSE.md "release/${STAGE}/LICENSE" 2>/dev/null || true+ (cd release && tar -czf "${STAGE}.tar.gz" "${STAGE}")+ # Bare binary tarball — single file (cua-driver) at the archive root,+ # mirroring the Swift cua-driver `*-binary.tar.gz` convention.+ tar -czf "release/${STAGE}-binary.tar.gz" -C "release/${STAGE}" cua-driver+ ls -lh "release/${STAGE}.tar.gz" "release/${STAGE}-binary.tar.gz"+ - uses: actions/upload-artifact@v4+ with:+ name: cua-driver-rs-linux-x86_64+ path: libs/cua-driver-rs/release/cua-driver-rs-*-linux-x86_64*.tar.gz+ if-no-files-found: error++ # ── Windows x86_64 ───────────────────────────────────────────────────────────+ build-windows:+ name: windows-x86_64+ runs-on: windows-latest+ steps:+ - uses: actions/checkout@v4+ - name: Determine version+ id: version+ shell: bash+ run: |+ if [[ "$GITHUB_REF" == refs/tags/cua-driver-rs-v* ]]; then+ VERSION="${GITHUB_REF#refs/tags/cua-driver-rs-v}"+ else+ VERSION="${{ inputs.version }}"+ fi+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"+ - uses: dtolnay/rust-toolchain@stable+ with:+ targets: x86_64-pc-windows-msvc+ - uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ libs/cua-driver-rs/target+ key: rust-cua-driver-rs-windows-x86_64-${{ hashFiles('libs/cua-driver-rs/Cargo.lock', 'libs/cua-driver-rs/**/Cargo.toml') }}+ restore-keys: |+ rust-cua-driver-rs-windows-x86_64-+ - name: Build (release)+ working-directory: libs/cua-driver-rs+ run: cargo build -p cua-driver --release --target x86_64-pc-windows-msvc+ - name: Package+ working-directory: libs/cua-driver-rs+ shell: pwsh+ run: |+ $version = "${{ steps.version.outputs.version }}"+ $label = "windows-x86_64"+ $stage = "cua-driver-rs-${version}-${label}"+ New-Item -ItemType Directory -Force -Path "release/$stage" | Out-Null+ Copy-Item "target/x86_64-pc-windows-msvc/release/cua-driver.exe" "release/$stage/"+ if (Test-Path "../../LICENSE.md") { Copy-Item "../../LICENSE.md" "release/$stage/LICENSE" }+ Compress-Archive -Path "release/$stage/*" -DestinationPath "release/$stage.zip" -Force+ # Bare binary (cua-driver.exe by itself) — matches Swift's+ # `*-binary.tar.gz` convention but as .zip since Windows tools+ # don't always have tar on PATH.+ Compress-Archive -Path "release/$stage/cua-driver.exe" -DestinationPath "release/$stage-binary.zip" -Force+ Get-ChildItem "release/$stage*.zip"+ - uses: actions/upload-artifact@v4+ with:+ name: cua-driver-rs-windows-x86_64+ path: libs/cua-driver-rs/release/cua-driver-rs-*-windows-x86_64*.zip+ if-no-files-found: error++ # ── macOS universal (arm64 + x86_64 → lipo) ─────────────────────────────────+ # Mirrors cd-swift-cua-driver.yml's approach: build both arches in one job+ # on macos-15, then combine into a universal binary via `lipo -create`.+ # The same universal binary is used in BOTH the arm64 and x86_64 named+ # tarballs (so callers that download by arch still get the universal+ # binary), plus a third `darwin-universal` tarball and a bare binary.+ build-macos-universal:+ name: darwin-universal+ runs-on: macos-15+ steps:+ - uses: actions/checkout@v4+ - name: Determine version+ id: version+ shell: bash+ run: |+ if [[ "$GITHUB_REF" == refs/tags/cua-driver-rs-v* ]]; then+ VERSION="${GITHUB_REF#refs/tags/cua-driver-rs-v}"+ else+ VERSION="${{ inputs.version }}"+ fi+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"+ - uses: dtolnay/rust-toolchain@stable+ with:+ targets: aarch64-apple-darwin, x86_64-apple-darwin+ - uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ libs/cua-driver-rs/target+ key: rust-cua-driver-rs-darwin-universal-${{ hashFiles('libs/cua-driver-rs/Cargo.lock', 'libs/cua-driver-rs/**/Cargo.toml') }}+ restore-keys: |+ rust-cua-driver-rs-darwin-universal-+ - name: Build arm64+ working-directory: libs/cua-driver-rs+ run: cargo build -p cua-driver --release --target aarch64-apple-darwin+ - name: Build x86_64+ working-directory: libs/cua-driver-rs+ run: cargo build -p cua-driver --release --target x86_64-apple-darwin+ - name: Create universal binary (lipo)+ working-directory: libs/cua-driver-rs+ run: |+ ARM64="target/aarch64-apple-darwin/release/cua-driver"+ X86="target/x86_64-apple-darwin/release/cua-driver"+ mkdir -p release/universal+ lipo -create "$ARM64" "$X86" -output release/universal/cua-driver+ lipo -info release/universal/cua-driver+ - name: Package+ working-directory: libs/cua-driver-rs+ run: |+ VERSION="${{ steps.version.outputs.version }}"+ for LABEL in darwin-arm64 darwin-x86_64 darwin-universal; do+ STAGE="cua-driver-rs-${VERSION}-${LABEL}"+ mkdir -p "release/${STAGE}"+ # Same universal binary in all three named tarballs — callers+ # that download by arch still get the universal slice+ # (matches the Swift `cd-swift-cua-driver.yml` convention).+ cp release/universal/cua-driver "release/${STAGE}/"+ cp ../../LICENSE.md "release/${STAGE}/LICENSE" 2>/dev/null || true+ (cd release && tar -czf "${STAGE}.tar.gz" "${STAGE}")+ done+ # Bare universal binary — single-file tarball matching Swift's+ # `cua-driver-${VERSION}-darwin-universal-binary.tar.gz`.+ tar -czf "release/cua-driver-rs-${VERSION}-darwin-universal-binary.tar.gz" \+ -C release/universal cua-driver+ ls -lh release/*.tar.gz+ - uses: actions/upload-artifact@v4+ with:+ name: cua-driver-rs-darwin+ path: libs/cua-driver-rs/release/cua-driver-rs-*-darwin-*.tar.gz+ if-no-files-found: error++ # ── Release ──────────────────────────────────────────────────────────────────+ release:+ needs: [build-linux, build-windows, build-macos-universal]+ if: startsWith(github.ref, 'refs/tags/cua-driver-rs-v')+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4+ with:+ fetch-depth: 0++ - name: Determine version+ id: version+ run: |+ VERSION="${GITHUB_REF#refs/tags/cua-driver-rs-v}"+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"++ - name: Download all artifacts+ uses: actions/download-artifact@v4+ with:+ path: artifacts++ - name: Stage release files+ run: |+ mkdir -p release-upload+ find artifacts -type f \( -name "*.tar.gz" -o -name "*.zip" \) \+ -exec cp {} release-upload/ \;+ echo "Release files:"+ ls -lh release-upload/++ - name: Generate SHA256 checksums+ id: checksums+ run: |+ cd release-upload+ {+ echo "## SHA256 Checksums"+ echo '```'+ shasum -a 256 cua-driver-rs-*.{tar.gz,zip} 2>/dev/null | sort+ echo '```'+ } > checksums.txt+ checksums=$(cat checksums.txt)+ echo "checksums<<EOF" >> "$GITHUB_OUTPUT"+ echo "$checksums" >> "$GITHUB_OUTPUT"+ echo "EOF" >> "$GITHUB_OUTPUT"++ - name: Generate path-filtered release notes+ id: notes+ run: |+ PREV_TAG=$(git tag -l "cua-driver-rs-v*" --sort=-v:refname \+ | grep -v "^${{ github.ref_name }}$" \+ | head -n 1 || echo "")+ if [ -n "$PREV_TAG" ]; then+ NOTES=$(git log ${PREV_TAG}..HEAD --pretty=format:"* %s (%h) by @%an" -- "libs/cua-driver-rs" | head -50)+ else+ NOTES=$(git log --pretty=format:"* %s (%h) by @%an" -- "libs/cua-driver-rs" | head -50)+ fi+ [ -z "$NOTES" ] && NOTES="* No path-specific changes found"+ {+ echo "RELEASE_NOTES<<EOF"+ echo "## What's Changed"+ echo ""+ echo "$NOTES"+ echo "EOF"+ } >> "$GITHUB_OUTPUT"++ - name: Create GitHub Release+ uses: softprops/action-gh-release@v2+ with:+ files: release-upload/*+ # The `prerelease: true` flag + the explicit BETA banner make it+ # clear to anyone landing on the release page that cua-driver-rs+ # is the cross-platform Rust port of the Swift cua-driver and is+ # still in BETA — the Swift cua-driver remains the production+ # macOS binary.+ prerelease: true+ body: |+ > ⚠️ **BETA — cua-driver-rs**+ >+ > `cua-driver-rs` is the **cross-platform Rust port** of the original Swift+ > [`cua-driver`](https://github.com/trycua/cua/tree/main/libs/cua-driver),+ > introducing **Windows and Linux support** to the cua-driver tool surface.+ >+ > - **Windows** and **Linux**: feature-complete, recommended for testing /+ > integration work.+ > - **macOS**: parity with the Swift cua-driver is in progress; for production+ > macOS automation prefer the Swift binary+ > (`cua-driver-v*` releases, installed via the Swift `install.sh`).+ >+ > The Rust port and the Swift driver publish to the same GitHub repo under+ > **distinct tag prefixes** (`cua-driver-rs-v*` vs `cua-driver-v*`), so their+ > installers do not collide.++ ${{ steps.notes.outputs.RELEASE_NOTES }}++ ${{ steps.checksums.outputs.checksums }}++ ### Install (Linux / macOS)++ ```bash+ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver-rs/scripts/install.sh)"+ ```++ ### Install (Windows)++ Download `cua-driver-rs-${{ steps.version.outputs.version }}-windows-x86_64-binary.zip` from the+ assets below, unzip to a location on your PATH (`%LOCALAPPDATA%\Programs\cua-driver-rs\`+ is the convention), then run `cua-driver.exe list-tools` to confirm.++ ### Artifacts++ **macOS (universal — arm64 + x86_64 in one binary, like the Swift cua-driver)**+ - `cua-driver-rs-${{ steps.version.outputs.version }}-darwin-universal.tar.gz` — directory tarball with LICENSE+ - `cua-driver-rs-${{ steps.version.outputs.version }}-darwin-arm64.tar.gz` — same universal binary, named for arm64 callers+ - `cua-driver-rs-${{ steps.version.outputs.version }}-darwin-x86_64.tar.gz` — same universal binary, named for x86_64 callers+ - `cua-driver-rs-${{ steps.version.outputs.version }}-darwin-universal-binary.tar.gz` — bare universal binary (single file at archive root)++ **Linux**+ - `cua-driver-rs-${{ steps.version.outputs.version }}-linux-x86_64.tar.gz` — directory tarball+ - `cua-driver-rs-${{ steps.version.outputs.version }}-linux-x86_64-binary.tar.gz` — bare binary++ **Windows**+ - `cua-driver-rs-${{ steps.version.outputs.version }}-windows-x86_64.zip` — directory zip+ - `cua-driver-rs-${{ steps.version.outputs.version }}-windows-x86_64-binary.zip` — bare `cua-driver.exe`+ generate_release_notes: false+ make_latest: falsediff --git a/.github/workflows/release-bump-version.yml b/.github/workflows/release-bump-version.ymlindex 74eb6ad718..6cb7208ea3 100644--- a/.github/workflows/release-bump-version.yml+++ b/.github/workflows/release-bump-version.yml@@ -28,6 +28,7 @@ on:- npm/cuabot- lume- cua-driver+ - cua-driver-rs- docker/cuabot- docker/kasm- docker/xfce@@ -43,6 +44,11 @@ on:- patch- minor- major+ target_branch:+ description: "Branch to push the bump commit + tag to"+ required: false+ type: string+ default: "main"permissions:contents: write@@ -135,6 +141,10 @@ jobs:echo "directory=libs/cua-driver" >> $GITHUB_OUTPUTecho "type=cua-driver" >> $GITHUB_OUTPUT;;+ "cua-driver-rs")+ echo "directory=libs/cua-driver-rs" >> $GITHUB_OUTPUT+ echo "type=cua-driver-rs" >> $GITHUB_OUTPUT+ ;;"docker/cuabot")echo "directory=libs/cuabot" >> $GITHUB_OUTPUTecho "type=docker" >> $GITHUB_OUTPUT@@ -174,6 +184,7 @@ jobs:uses: actions/checkout@v4with:fetch-depth: 0+ ref: ${{ inputs.target_branch || 'main' }}token: ${{ secrets.GITHUB_TOKEN }}- name: Set up Python@@ -393,6 +404,14 @@ jobs:echo "cua-driver version: $VERSION"echo "version=$VERSION" >> "$GITHUB_OUTPUT"+ - name: Capture bumped cua-driver-rs version+ if: ${{ inputs.service == 'cua-driver-rs' }}+ id: cua_driver_rs_version+ run: |+ VERSION=$(grep -m1 -oP '^version = "\K[^"]+' libs/cua-driver-rs/Cargo.toml)+ echo "cua-driver-rs version: $VERSION"+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"+- name: Capture bumped cuabot versionif: ${{ inputs.service == 'npm/cuabot' }}id: cuabot_version@@ -410,9 +429,10 @@ jobs:owner: ${{ github.repository_owner }}repositories: ${{ github.event.repository.name }}- - name: Push release to main+ - name: Push release to target branchenv:GH_TOKEN: ${{ steps.app-token.outputs.token }}+ TARGET_BRANCH: ${{ inputs.target_branch || 'main' }}run: |# Configure git to use the app tokengit remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git"@@ -439,15 +459,15 @@ jobs:echo ""echo "=== Attempt $RETRY_COUNT/$MAX_RETRIES ==="- # Fetch latest main to check if we need to rebase- git fetch origin main+ # Fetch latest target branch to check if we need to rebase+ git fetch origin "${TARGET_BRANCH}"- # Check if our HEAD is a descendant of origin/main (fast-forward possible)- if ! git merge-base --is-ancestor origin/main HEAD; then- echo "Main branch has moved forward, rebasing..."+ # Check if our HEAD is a descendant of origin/<branch> (fast-forward possible)+ if ! git merge-base --is-ancestor "origin/${TARGET_BRANCH}" HEAD; then+ echo "Branch ${TARGET_BRANCH} has moved forward, rebasing..."- # Rebase our commit(s) onto the latest main- git rebase origin/main+ # Rebase our commit(s) onto the latest branch+ git rebase "origin/${TARGET_BRANCH}"# Update the tag to point to the rebased commitfor tag in $ALL_TAGS; do@@ -464,13 +484,13 @@ jobs:echo "Pushing to temp branch..."git push origin "HEAD:refs/heads/${TEMP_BRANCH}" --force- # Try to update main branch via API- echo "Updating main branch via API..."- if gh api -X PATCH /repos/${{ github.repository }}/git/refs/heads/main \+ # Try to update target branch via API+ echo "Updating ${TARGET_BRANCH} branch via API..."+ if gh api -X PATCH "/repos/${{ github.repository }}/git/refs/heads/${TARGET_BRANCH}" \-f sha="${COMMIT_SHA}" \-F force=false 2>&1; then- echo "Successfully updated main branch!"+ echo "Successfully updated ${TARGET_BRANCH} branch!"# Create tag via API (only after main is successfully updated)echo "Creating tag via API..."@@ -490,11 +510,11 @@ jobs:exit 0fi- echo "Failed to update main (not a fast-forward), will retry..."+ echo "Failed to update ${TARGET_BRANCH} (not a fast-forward), will retry..."sleep 2done- echo "ERROR: Failed to update main branch after $MAX_RETRIES attempts"+ echo "ERROR: Failed to update ${TARGET_BRANCH} branch after $MAX_RETRIES attempts"exit 1# NOTE: Publishing is handled by tag-triggered workflows (cd-py-*.yml, cd-ts-*.yml, cd-swift-lume.yml)diff --git a/libs/cua-driver-rs/.bumpversion.cfg b/libs/cua-driver-rs/.bumpversion.cfgnew file mode 100644index 0000000000..036fb04528--- /dev/null+++ b/libs/cua-driver-rs/.bumpversion.cfg@@ -0,0 +1,10 @@+[bumpversion]+current_version = 0.1.2+commit = True+tag = True+tag_name = cua-driver-rs-v{new_version}+message = Bump cua-driver-rs to v{new_version}++[bumpversion:file:Cargo.toml]+search = version = "{current_version}"+replace = version = "{new_version}"diff --git a/libs/cua-driver-rs/.cargo/config.toml b/libs/cua-driver-rs/.cargo/config.tomlnew file mode 100644index 0000000000..ac2b23f814--- /dev/null+++ b/libs/cua-driver-rs/.cargo/config.toml@@ -0,0 +1,2 @@+[target.x86_64-pc-windows-msvc]+rustflags = ["-C", "target-feature=+crt-static"]diff --git a/libs/cua-driver-rs/.gitignore b/libs/cua-driver-rs/.gitignorenew file mode 100644index 0000000000..d70eb15e28--- /dev/null+++ b/libs/cua-driver-rs/.gitignore@@ -0,0 +1,3 @@+# Large test assets — Electron app bundle exceeds GitHub's 100 MB file limit.+# Download separately for local integration testing.+tests/integration/v2/assets/electron/diff --git a/libs/cua-driver-rs/Cargo.lock b/libs/cua-driver-rs/Cargo.locknew file mode 100644index 0000000000..1779741749--- /dev/null+++ b/libs/cua-driver-rs/Cargo.lock@@ -0,0 +1,2049 @@+# This file is automatically @generated by Cargo.+# It is not intended for manual editing.+version = 4++[[package]]+name = "adler2"+version = "2.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"++[[package]]+name = "aho-corasick"+version = "1.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"+dependencies = [+ "memchr",+]++[[package]]+name = "anyhow"+version = "1.0.102"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"++[[package]]+name = "arrayref"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"++[[package]]+name = "arrayvec"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"++[[package]]+name = "async-trait"+version = "0.1.89"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "autocfg"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"++[[package]]+name = "base64"+version = "0.22.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"++[[package]]+name = "bitflags"+version = "1.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"++[[package]]+name = "bitflags"+version = "2.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"++[[package]]+name = "block-buffer"+version = "0.10.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"+dependencies = [+ "generic-array",+]++[[package]]+name = "block2"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f"+dependencies = [+ "objc2",+]++[[package]]+name = "bytemuck"+version = "1.25.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"++[[package]]+name = "byteorder"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"++[[package]]+name = "byteorder-lite"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"++[[package]]+name = "bytes"+version = "1.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"++[[package]]+name = "cc"+version = "1.2.62"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"+dependencies = [+ "find-msvc-tools",+ "shlex",+]++[[package]]+name = "cfg-if"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"++[[package]]+name = "core-foundation"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "core-foundation-sys"+version = "0.8.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"++[[package]]+name = "core-graphics"+version = "0.24.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1"+dependencies = [+ "bitflags 2.11.1",+ "core-foundation",+ "core-graphics-types",+ "foreign-types 0.5.0",+ "libc",+]++[[package]]+name = "core-graphics-types"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb"+dependencies = [+ "bitflags 2.11.1",+ "core-foundation",+ "libc",+]++[[package]]+name = "core_maths"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30"+dependencies = [+ "libm",+]++[[package]]+name = "cpufeatures"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"+dependencies = [+ "libc",+]++[[package]]+name = "crc32fast"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"+dependencies = [+ "cfg-if",+]++[[package]]+name = "crypto-common"+version = "0.1.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"+dependencies = [+ "generic-array",+ "typenum",+]++[[package]]+name = "cua-driver"+version = "0.1.2"+dependencies = [+ "anyhow",+ "async-trait",+ "base64",+ "cursor-overlay",+ "image",+ "mcp-server",+ "platform-linux",+ "platform-macos",+ "platform-windows",+ "serde",+ "serde_json",+ "tokio",+ "tracing",+ "tracing-subscriber",+]++[[package]]+name = "cursor-overlay"+version = "0.1.2"+dependencies = [+ "anyhow",+ "image",+ "resvg",+ "serde",+ "serde_json",+ "tiny-skia",+ "tracing",+ "usvg",+]++[[package]]+name = "data-encoding"+version = "2.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"++[[package]]+name = "data-url"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376"++[[package]]+name = "digest"+version = "0.10.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"+dependencies = [+ "block-buffer",+ "crypto-common",+]++[[package]]+name = "equivalent"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"++[[package]]+name = "errno"+version = "0.3.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"+dependencies = [+ "libc",+ "windows-sys",+]++[[package]]+name = "euclid"+version = "0.22.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06"+dependencies = [+ "num-traits",+]++[[package]]+name = "fastrand"+version = "2.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"++[[package]]+name = "fdeflate"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"+dependencies = [+ "simd-adler32",+]++[[package]]+name = "find-msvc-tools"+version = "0.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"++[[package]]+name = "flate2"+version = "1.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"+dependencies = [+ "crc32fast",+ "miniz_oxide",+]++[[package]]+name = "float-cmp"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"++[[package]]+name = "focus-monitor-win"+version = "0.1.2"+dependencies = [+ "windows",+]++[[package]]+name = "foldhash"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"++[[package]]+name = "fontdb"+version = "0.22.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a3a6f9af55fb97ad673fb7a69533eb2f967648a06fa21f8c9bb2cd6d33975716"+dependencies = [+ "log",+ "slotmap",+ "tinyvec",+ "ttf-parser",+]++[[package]]+name = "foreign-types"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"+dependencies = [+ "foreign-types-shared 0.1.1",+]++[[package]]+name = "foreign-types"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"+dependencies = [+ "foreign-types-macros",+ "foreign-types-shared 0.3.1",+]++[[package]]+name = "foreign-types-macros"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "foreign-types-shared"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"++[[package]]+name = "foreign-types-shared"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"++[[package]]+name = "futures-core"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"++[[package]]+name = "futures-macro"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "futures-sink"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"++[[package]]+name = "futures-task"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"++[[package]]+name = "futures-util"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"+dependencies = [+ "futures-core",+ "futures-macro",+ "futures-sink",+ "futures-task",+ "pin-project-lite",+ "slab",+]++[[package]]+name = "generic-array"+version = "0.14.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"+dependencies = [+ "typenum",+ "version_check",+]++[[package]]+name = "gethostname"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8"+dependencies = [+ "rustix",+ "windows-link",+]++[[package]]+name = "getrandom"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"+dependencies = [+ "cfg-if",+ "libc",+ "wasi",+]++[[package]]+name = "getrandom"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"+dependencies = [+ "cfg-if",+ "libc",+ "r-efi",+ "wasip2",+ "wasip3",+]++[[package]]+name = "hashbrown"+version = "0.15.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"+dependencies = [+ "foldhash",+]++[[package]]+name = "hashbrown"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"++[[package]]+name = "heck"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"++[[package]]+name = "http"+version = "1.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"+dependencies = [+ "bytes",+ "itoa",+]++[[package]]+name = "httparse"+version = "1.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"++[[package]]+name = "id-arena"+version = "2.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"++[[package]]+name = "image"+version = "0.25.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"+dependencies = [+ "bytemuck",+ "byteorder-lite",+ "moxcms",+ "num-traits",+ "png 0.18.1",+ "zune-core",+ "zune-jpeg",+]++[[package]]+name = "imagesize"+version = "0.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "edcd27d72f2f071c64249075f42e205ff93c9a4c5f6c6da53e79ed9f9832c285"++[[package]]+name = "indexmap"+version = "2.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"+dependencies = [+ "equivalent",+ "hashbrown 0.17.1",+ "serde",+ "serde_core",+]++[[package]]+name = "itoa"+version = "1.0.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"++[[package]]+name = "kurbo"+version = "0.11.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c62026ae44756f8a599ba21140f350303d4f08dcdcc71b5ad9c9bb8128c13c62"+dependencies = [+ "arrayvec",+ "euclid",+ "smallvec",+]++[[package]]+name = "lazy_static"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"++[[package]]+name = "leb128fmt"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"++[[package]]+name = "libc"+version = "0.2.186"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"++[[package]]+name = "libm"+version = "0.2.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"++[[package]]+name = "linux-raw-sys"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"++[[package]]+name = "lock_api"+version = "0.4.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"+dependencies = [+ "scopeguard",+]++[[package]]+name = "log"+version = "0.4.29"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"++[[package]]+name = "matchers"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"+dependencies = [+ "regex-automata",+]++[[package]]+name = "mcp-server"+version = "0.1.2"+dependencies = [+ "anyhow",+ "async-trait",+ "serde",+ "serde_json",+ "thiserror",+ "tokio",+ "tracing",+]++[[package]]+name = "memchr"+version = "2.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"++[[package]]+name = "miniz_oxide"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"+dependencies = [+ "adler2",+ "simd-adler32",+]++[[package]]+name = "mio"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"+dependencies = [+ "libc",+ "wasi",+ "windows-sys",+]++[[package]]+name = "moxcms"+version = "0.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"+dependencies = [+ "num-traits",+ "pxfm",+]++[[package]]+name = "native-tls"+version = "0.2.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"+dependencies = [+ "libc",+ "log",+ "openssl",+ "openssl-probe",+ "openssl-sys",+ "schannel",+ "security-framework",+ "security-framework-sys",+ "tempfile",+]++[[package]]+name = "nu-ansi-term"+version = "0.50.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"+dependencies = [+ "windows-sys",+]++[[package]]+name = "num-traits"+version = "0.2.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"+dependencies = [+ "autocfg",+]++[[package]]+name = "objc-sys"+version = "0.3.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310"++[[package]]+name = "objc2"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804"+dependencies = [+ "objc-sys",+ "objc2-encode",+]++[[package]]+name = "objc2-app-kit"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff"+dependencies = [+ "bitflags 2.11.1",+ "block2",+ "libc",+ "objc2",+ "objc2-core-data",+ "objc2-core-image",+ "objc2-foundation",+ "objc2-quartz-core",+]++[[package]]+name = "objc2-core-data"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef"+dependencies = [+ "bitflags 2.11.1",+ "block2",+ "objc2",+ "objc2-foundation",+]++[[package]]+name = "objc2-core-image"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80"+dependencies = [+ "block2",+ "objc2",+ "objc2-foundation",+ "objc2-metal",+]++[[package]]+name = "objc2-encode"+version = "4.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33"++[[package]]+name = "objc2-foundation"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8"+dependencies = [+ "bitflags 2.11.1",+ "block2",+ "libc",+ "objc2",+]++[[package]]+name = "objc2-metal"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6"+dependencies = [+ "bitflags 2.11.1",+ "block2",+ "objc2",+ "objc2-foundation",+]++[[package]]+name = "objc2-quartz-core"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a"+dependencies = [+ "bitflags 2.11.1",+ "block2",+ "objc2",+ "objc2-foundation",+ "objc2-metal",+]++[[package]]+name = "once_cell"+version = "1.21.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"++[[package]]+name = "openssl"+version = "0.10.79"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542"+dependencies = [+ "bitflags 2.11.1",+ "cfg-if",+ "foreign-types 0.3.2",+ "libc",+ "openssl-macros",+ "openssl-sys",+]++[[package]]+name = "openssl-macros"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "openssl-probe"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"++[[package]]+name = "openssl-sys"+version = "0.9.115"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781"+dependencies = [+ "cc",+ "libc",+ "pkg-config",+ "vcpkg",+]++[[package]]+name = "parking_lot"+version = "0.12.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"+dependencies = [+ "lock_api",+ "parking_lot_core",+]++[[package]]+name = "parking_lot_core"+version = "0.9.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"+dependencies = [+ "cfg-if",+ "libc",+ "redox_syscall",+ "smallvec",+ "windows-link",+]++[[package]]+name = "pico-args"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315"++[[package]]+name = "pin-project-lite"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"++[[package]]+name = "pkg-config"+version = "0.3.33"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"++[[package]]+name = "platform-linux"+version = "0.1.2"+dependencies = [+ "anyhow",+ "async-trait",+ "base64",+ "cursor-overlay",+ "image",+ "mcp-server",+ "serde",+ "serde_json",+ "thiserror",+ "tiny-skia",+ "tokio",+ "tracing",+ "x11rb",+]++[[package]]+name = "platform-macos"+version = "0.1.2"+dependencies = [+ "anyhow",+ "async-trait",+ "base64",+ "core-foundation",+ "core-graphics",+ "cursor-overlay",+ "foreign-types 0.5.0",+ "futures-util",+ "image",+ "libc",+ "mcp-server",+ "objc2",+ "objc2-app-kit",+ "objc2-foundation",+ "objc2-quartz-core",+ "serde",+ "serde_json",+ "thiserror",+ "tiny-skia",+ "tokio",+ "tokio-tungstenite",+ "tracing",+]++[[package]]+name = "platform-windows"+version = "0.1.2"+dependencies = [+ "anyhow",+ "async-trait",+ "base64",+ "cursor-overlay",+ "image",+ "mcp-server",+ "serde",+ "serde_json",+ "thiserror",+ "tiny-skia",+ "tokio",+ "tracing",+ "windows",+]++[[package]]+name = "png"+version = "0.17.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526"+dependencies = [+ "bitflags 1.3.2",+ "crc32fast",+ "fdeflate",+ "flate2",+ "miniz_oxide",+]++[[package]]+name = "png"+version = "0.18.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"+dependencies = [+ "bitflags 2.11.1",+ "crc32fast",+ "fdeflate",+ "flate2",+ "miniz_oxide",+]++[[package]]+name = "ppv-lite86"+version = "0.2.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"+dependencies = [+ "zerocopy",+]++[[package]]+name = "prettyplease"+version = "0.2.37"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"+dependencies = [+ "proc-macro2",+ "syn",+]++[[package]]+name = "proc-macro2"+version = "1.0.106"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "pxfm"+version = "0.1.29"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f"++[[package]]+name = "quote"+version = "1.0.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"+dependencies = [+ "proc-macro2",+]++[[package]]+name = "r-efi"+version = "6.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"++[[package]]+name = "rand"+version = "0.8.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"+dependencies = [+ "libc",+ "rand_chacha",+ "rand_core",+]++[[package]]+name = "rand_chacha"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"+dependencies = [+ "ppv-lite86",+ "rand_core",+]++[[package]]+name = "rand_core"+version = "0.6.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"+dependencies = [+ "getrandom 0.2.17",+]++[[package]]+name = "redox_syscall"+version = "0.5.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"+dependencies = [+ "bitflags 2.11.1",+]++[[package]]+name = "regex-automata"+version = "0.4.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",+]++[[package]]+name = "regex-syntax"+version = "0.8.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"++[[package]]+name = "resvg"+version = "0.44.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4a325d5e8d1cebddd070b13f44cec8071594ab67d1012797c121f27a669b7958"+dependencies = [+ "log",+ "pico-args",+ "rgb",+ "svgtypes",+ "tiny-skia",+ "usvg",+]++[[package]]+name = "rgb"+version = "0.8.53"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"+dependencies = [+ "bytemuck",+]++[[package]]+name = "roxmltree"+version = "0.20.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"++[[package]]+name = "rustix"+version = "1.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"+dependencies = [+ "bitflags 2.11.1",+ "errno",+ "libc",+ "linux-raw-sys",+ "windows-sys",+]++[[package]]+name = "rustybuzz"+version = "0.18.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c85d1ccd519e61834798eb52c4e886e8c2d7d698dd3d6ce0b1b47eb8557f1181"+dependencies = [+ "bitflags 2.11.1",+ "bytemuck",+ "core_maths",+ "log",+ "smallvec",+ "ttf-parser",+ "unicode-bidi-mirroring",+ "unicode-ccc",+ "unicode-properties",+ "unicode-script",+]++[[package]]+name = "schannel"+version = "0.1.29"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"+dependencies = [+ "windows-sys",+]++[[package]]+name = "scopeguard"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"++[[package]]+name = "security-framework"+version = "3.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"+dependencies = [+ "bitflags 2.11.1",+ "core-foundation",+ "core-foundation-sys",+ "libc",+ "security-framework-sys",+]++[[package]]+name = "security-framework-sys"+version = "2.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "semver"+version = "1.0.28"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"++[[package]]+name = "serde"+version = "1.0.228"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"+dependencies = [+ "serde_core",+ "serde_derive",+]++[[package]]+name = "serde_core"+version = "1.0.228"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"+dependencies = [+ "serde_derive",+]++[[package]]+name = "serde_derive"+version = "1.0.228"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "serde_json"+version = "1.0.149"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"+dependencies = [+ "itoa",+ "memchr",+ "serde",+ "serde_core",+ "zmij",+]++[[package]]+name = "sha1"+version = "0.10.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"+dependencies = [+ "cfg-if",+ "cpufeatures",+ "digest",+]++[[package]]+name = "sharded-slab"+version = "0.1.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"+dependencies = [+ "lazy_static",+]++[[package]]+name = "shlex"+version = "1.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"++[[package]]+name = "signal-hook-registry"+version = "1.4.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"+dependencies = [+ "errno",+ "libc",+]++[[package]]+name = "simd-adler32"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"++[[package]]+name = "simplecss"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7a9c6883ca9c3c7c90e888de77b7a5c849c779d25d74a1269b0218b14e8b136c"+dependencies = [+ "log",+]++[[package]]+name = "siphasher"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"++[[package]]+name = "slab"+version = "0.4.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"++[[package]]+name = "slotmap"+version = "1.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038"+dependencies = [+ "version_check",+]++[[package]]+name = "smallvec"+version = "1.15.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"++[[package]]+name = "socket2"+version = "0.6.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"+dependencies = [+ "libc",+ "windows-sys",+]++[[package]]+name = "strict-num"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731"+dependencies = [+ "float-cmp",+]++[[package]]+name = "svgtypes"+version = "0.15.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68c7541fff44b35860c1a7a47a7cadf3e4a304c457b58f9870d9706ece028afc"+dependencies = [+ "kurbo",+ "siphasher",+]++[[package]]+name = "syn"+version = "2.0.117"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "tempfile"+version = "3.27.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"+dependencies = [+ "fastrand",+ "getrandom 0.4.2",+ "once_cell",+ "rustix",+ "windows-sys",+]++[[package]]+name = "thiserror"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"+dependencies = [+ "thiserror-impl",+]++[[package]]+name = "thiserror-impl"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "thread_local"+version = "1.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"+dependencies = [+ "cfg-if",+]++[[package]]+name = "tiny-skia"+version = "0.11.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "83d13394d44dae3207b52a326c0c85a8bf87f1541f23b0d143811088497b09ab"+dependencies = [+ "arrayref",+ "arrayvec",+ "bytemuck",+ "cfg-if",+ "log",+ "png 0.17.16",+ "tiny-skia-path",+]++[[package]]+name = "tiny-skia-path"+version = "0.11.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9c9e7fc0c2e86a30b117d0462aa261b72b7a99b7ebd7deb3a14ceda95c5bdc93"+dependencies = [+ "arrayref",+ "bytemuck",+ "strict-num",+]++[[package]]+name = "tinyvec"+version = "1.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"+dependencies = [+ "tinyvec_macros",+]++[[package]]+name = "tinyvec_macros"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"++[[package]]+name = "tokio"+version = "1.52.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6"+dependencies = [+ "bytes",+ "libc",+ "mio",+ "parking_lot",+ "pin-project-lite",+ "signal-hook-registry",+ "socket2",+ "tokio-macros",+ "windows-sys",+]++[[package]]+name = "tokio-macros"+version = "2.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "tokio-native-tls"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"+dependencies = [+ "native-tls",+ "tokio",+]++[[package]]+name = "tokio-tungstenite"+version = "0.24.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"+dependencies = [+ "futures-util",+ "log",+ "native-tls",+ "tokio",+ "tokio-native-tls",+ "tungstenite",+]++[[package]]+name = "tracing"+version = "0.1.44"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"+dependencies = [+ "pin-project-lite",+ "tracing-attributes",+ "tracing-core",+]++[[package]]+name = "tracing-attributes"+version = "0.1.31"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "tracing-core"+version = "0.1.36"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"+dependencies = [+ "once_cell",+ "valuable",+]++[[package]]+name = "tracing-log"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"+dependencies = [+ "log",+ "once_cell",+ "tracing-core",+]++[[package]]+name = "tracing-subscriber"+version = "0.3.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"+dependencies = [+ "matchers",+ "nu-ansi-term",+ "once_cell",+ "regex-automata",+ "sharded-slab",+ "smallvec",+ "thread_local",+ "tracing",+ "tracing-core",+ "tracing-log",+]++[[package]]+name = "ttf-parser"+version = "0.24.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5be21190ff5d38e8b4a2d3b6a3ae57f612cc39c96e83cedeaf7abc338a8bac4a"+dependencies = [+ "core_maths",+]++[[package]]+name = "tungstenite"+version = "0.24.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"+dependencies = [+ "byteorder",+ "bytes",+ "data-encoding",+ "http",+ "httparse",+ "log",+ "native-tls",+ "rand",+ "sha1",+ "thiserror",+ "utf-8",+]++[[package]]+name = "typenum"+version = "1.20.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"++[[package]]+name = "unicode-bidi"+version = "0.3.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"++[[package]]+name = "unicode-bidi-mirroring"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "64af057ad7466495ca113126be61838d8af947f41d93a949980b2389a118082f"++[[package]]+name = "unicode-ccc"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "260bc6647b3893a9a90668360803a15f96b85a5257b1c3a0c3daf6ae2496de42"++[[package]]+name = "unicode-ident"+version = "1.0.24"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"++[[package]]+name = "unicode-properties"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"++[[package]]+name = "unicode-script"+version = "0.5.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee"++[[package]]+name = "unicode-vo"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94"++[[package]]+name = "unicode-xid"+version = "0.2.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"++[[package]]+name = "usvg"+version = "0.44.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7447e703d7223b067607655e625e0dbca80822880248937da65966194c4864e6"+dependencies = [+ "base64",+ "data-url",+ "flate2",+ "fontdb",+ "imagesize",+ "kurbo",+ "log",+ "pico-args",+ "roxmltree",+ "rustybuzz",+ "simplecss",+ "siphasher",+ "strict-num",+ "svgtypes",+ "tiny-skia-path",+ "unicode-bidi",+ "unicode-script",+ "unicode-vo",+ "xmlwriter",+]++[[package]]+name = "utf-8"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"++[[package]]+name = "valuable"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"++[[package]]+name = "vcpkg"+version = "0.2.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"++[[package]]+name = "version_check"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"++[[package]]+name = "wasi"+version = "0.11.1+wasi-snapshot-preview1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"++[[package]]+name = "wasip2"+version = "1.0.3+wasi-0.2.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"+dependencies = [+ "wit-bindgen 0.57.1",+]++[[package]]+name = "wasip3"+version = "0.4.0+wasi-0.3.0-rc-2026-01-06"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"+dependencies = [+ "wit-bindgen 0.51.0",+]++[[package]]+name = "wasm-encoder"+version = "0.244.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"+dependencies = [+ "leb128fmt",+ "wasmparser",+]++[[package]]+name = "wasm-metadata"+version = "0.244.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"+dependencies = [+ "anyhow",+ "indexmap",+ "wasm-encoder",+ "wasmparser",+]++[[package]]+name = "wasmparser"+version = "0.244.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"+dependencies = [+ "bitflags 2.11.1",+ "hashbrown 0.15.5",+ "indexmap",+ "semver",+]++[[package]]+name = "windows"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"+dependencies = [+ "windows-core",+ "windows-targets",+]++[[package]]+name = "windows-core"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"+dependencies = [+ "windows-implement",+ "windows-interface",+ "windows-result",+ "windows-strings",+ "windows-targets",+]++[[package]]+name = "windows-implement"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-interface"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-link"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"++[[package]]+name = "windows-result"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"+dependencies = [+ "windows-targets",+]++[[package]]+name = "windows-strings"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"+dependencies = [+ "windows-result",+ "windows-targets",+]++[[package]]+name = "windows-sys"+version = "0.61.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"+dependencies = [+ "windows-link",+]++[[package]]+name = "windows-targets"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"+dependencies = [+ "windows_aarch64_gnullvm",+ "windows_aarch64_msvc",+ "windows_i686_gnu",+ "windows_i686_gnullvm",+ "windows_i686_msvc",+ "windows_x86_64_gnu",+ "windows_x86_64_gnullvm",+ "windows_x86_64_msvc",+]++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"++[[package]]+name = "windows_aarch64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"++[[package]]+name = "windows_i686_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"++[[package]]+name = "windows_i686_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"++[[package]]+name = "windows_i686_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"++[[package]]+name = "windows_x86_64_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"++[[package]]+name = "windows_x86_64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"++[[package]]+name = "wit-bindgen"+version = "0.51.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"+dependencies = [+ "wit-bindgen-rust-macro",+]++[[package]]+name = "wit-bindgen"+version = "0.57.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"++[[package]]+name = "wit-bindgen-core"+version = "0.51.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"+dependencies = [+ "anyhow",+ "heck",+ "wit-parser",+]++[[package]]+name = "wit-bindgen-rust"+version = "0.51.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"+dependencies = [+ "anyhow",+ "heck",+ "indexmap",+ "prettyplease",+ "syn",+ "wasm-metadata",+ "wit-bindgen-core",+ "wit-component",+]++[[package]]+name = "wit-bindgen-rust-macro"+version = "0.51.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"+dependencies = [+ "anyhow",+ "prettyplease",+ "proc-macro2",+ "quote",+ "syn",+ "wit-bindgen-core",+ "wit-bindgen-rust",+]++[[package]]+name = "wit-component"+version = "0.244.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"+dependencies = [+ "anyhow",+ "bitflags 2.11.1",+ "indexmap",+ "log",+ "serde",+ "serde_derive",+ "serde_json",+ "wasm-encoder",+ "wasm-metadata",+ "wasmparser",+ "wit-parser",+]++[[package]]+name = "wit-parser"+version = "0.244.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"+dependencies = [+ "anyhow",+ "id-arena",+ "indexmap",+ "log",+ "semver",+ "serde",+ "serde_derive",+ "serde_json",+ "unicode-xid",+ "wasmparser",+]++[[package]]+name = "x11rb"+version = "0.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414"+dependencies = [+ "gethostname",+ "rustix",+ "x11rb-protocol",+]++[[package]]+name = "x11rb-protocol"+version = "0.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd"++[[package]]+name = "xmlwriter"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9"++[[package]]+name = "zerocopy"+version = "0.8.48"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"+dependencies = [+ "zerocopy-derive",+]++[[package]]+name = "zerocopy-derive"+version = "0.8.48"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "zmij"+version = "1.0.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"++[[package]]+name = "zune-core"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"++[[package]]+name = "zune-jpeg"+version = "0.5.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"+dependencies = [+ "zune-core",+]diff --git a/libs/cua-driver-rs/Cargo.toml b/libs/cua-driver-rs/Cargo.tomlnew file mode 100644index 0000000000..e9a4105a27--- /dev/null+++ b/libs/cua-driver-rs/Cargo.toml@@ -0,0 +1,30 @@+[workspace]+resolver = "2"+members = [+ "crates/cua-driver",+ "crates/mcp-server",+ "crates/platform-macos",+ "crates/platform-windows",+ "crates/platform-linux",+ "crates/cursor-overlay",+ "crates/focus-monitor-win",+]++[workspace.package]+version = "0.1.2"+edition = "2021"+authors = ["trycua"]+license = "MIT"+repository = "https://github.com/trycua/cua"++[workspace.dependencies]+tokio = { version = "1", features = ["full"] }+serde = { version = "1", features = ["derive"] }+serde_json = "1"+anyhow = "1"+thiserror = "1"+tracing = "0.1"+tracing-subscriber = { version = "0.3", features = ["env-filter"] }+base64 = "0.22"+uuid = { version = "1", features = ["v4"] }+image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }diff --git a/libs/cua-driver-rs/PARITY.md b/libs/cua-driver-rs/PARITY.mdnew file mode 100644index 0000000000..9ff1dc1f6a--- /dev/null+++ b/libs/cua-driver-rs/PARITY.md@@ -0,0 +1,1235 @@+# cua-driver-rs ↔ cua-driver (Swift) Parity Audit++## Integration Test Suite++All API surfaces (CLI subcommands, stdio MCP protocol, daemon lifecycle) are+covered by `tests/integration/test_api_parity.py` — **111 tests per binary,+222 total**, parametrized so both binaries run the identical suite.++```bash+# Rust binary only+cd libs/cua-driver-rs/tests/integration+./run_tests.sh test_api_parity -v++# Both binaries side-by-side+./run_tests.sh --parity -v+```++### Known parity gaps (as of 2026-05-13, macOS)++| Gap | Rust | Swift |+|-----|------|-------|+| `type_text_chars` | ✅ | missing |+| `browser_eval` | ✅ | missing |+| `get_accessibility_tree` | ✅ | missing |+| `page` tool | ✅ (registered as of this commit) | ✅ |+| `--version` flag | ✅ (fixed in this commit) | ✅ |+| `call check_permissions` JSON | ✅ JSON | human-readable text |+| `call screenshot` (no window_id) | ✅ full-display default | error (requires window_id) |++---++Tracking surface-by-surface line-by-line behavioral comparison between the+Rust port and the Swift reference. Each entry lists Swift source location,+Rust source location, divergences (intentional vs. accidental), and the+deterministic test that locks the verified behavior in.++Format per entry:+```+## <surface>+- Swift: <path:line>+- Rust: macos=<path:line>, windows=<path:line>, linux=<path:line>+- Status: VERIFIED | INTENTIONAL_DIVERGENCE | OPEN+- Test: <path>+- Notes: ...+```++---++## MCP tool: `move_cursor`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/MoveCursorTool.swift:6-60`+- Rust:+ - macos=`crates/platform-macos/src/tools/move_cursor.rs`+ - windows=`crates/platform-windows/src/tools/impl_.rs` (MoveCursorTool)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (MoveCursorTool)+- Status: INTENTIONAL_DIVERGENCE (semantic) + VERIFIED (overlay behavior)+- Test: `crates/platform-windows/examples/cursor_visibility.rs`++### Intentional semantic divergence++Swift's `move_cursor` calls `CGWarpMouseCursorPosition` — it warps the+**real OS cursor** instantly. The Rust port repurposes the same tool name+to drive the **agent overlay** (animated, non-warping arrow) instead.++Rationale: the entire premise of `cua-driver-rs` is background automation+that never steals focus and never moves the user's physical mouse. Porting+the Swift cursor-warp behavior would directly violate Swift's own+focus-guard / no-cursor-warp invariants enforced elsewhere (click,+type_text, etc.). The Rust port treats `move_cursor` as "show the agent's+attention" — visual only.++Consequences:+- Schema accepts an extra optional `cursor_id: string` (multi-cursor support+ doesn't exist in Swift).+- Schema accepts floats (`number`) for `x`/`y`; Swift only accepts integers.+ Rust's looser type accepts every Swift-valid integer plus fractional pixel+ targets used by HiDPI flows.+- Response text uses `Agent cursor '<id>' moved to (X.X, Y.Y).` instead of+ `✅ Moved cursor to (X, Y).` — the Swift wording would be misleading+ given the different semantics.++### Cross-platform consistency (verified)++All three Rust platforms now send:+```+OverlayCommand::MoveTo { x, y, end_heading_radians: FRAC_PI_4 }+```++`FRAC_PI_4` (π/4) matches Swift `AgentCursor.animateAndWait(endAngleDegrees:+45)` so the overlay arrow always settles pointing upper-left. Linux was+previously sending `0.0` (left-pointing); fixed in this commit.++The deterministic test (`cursor_visibility.rs`) drives the live daemon via+the named pipe, sets a magenta gradient, sends `move_cursor`, and polls+screenshots until the cursor centroid settles. Asserts the final centroid+is within 100 px of the requested target and that ≥50 magenta pixels are+rendered. Hard 4 s timeout. Verified on Windows; should run on macOS/Linux+once the daemon pipe is exposed there (macOS uses Unix socket, Linux uses+Unix socket).++---++## MCP tool: `get_cursor_position`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/GetCursorPositionTool.swift:6-37`+- Rust:+ - macos=`crates/platform-macos/src/tools/get_cursor_position.rs`+ - windows=`crates/platform-windows/src/tools/impl_.rs` (GetCursorPositionTool)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (GetCursorPositionTool)+- Status: VERIFIED+- Test: `crates/platform-windows/examples/get_cursor_position_parity.rs`++### Fixed divergences++1. **Response text format** — Swift returns `"✅ Cursor at (X, Y)"`; Rust on+ every platform was returning `"Cursor: (X, Y)"` (no checkmark, wrong+ word). All three platforms now match Swift exactly.+2. **macOS coord type** — Swift truncates to `Int(pos.x)`; macOS Rust was+ returning floats formatted `"({x:.1}, {y:.1})"`. Now truncates to+ integers like Swift, consistent with Windows/Linux Rust.+3. **Description text** — was inconsistent across Rust platforms. All+ three now use Swift's wording: `"Return the current mouse cursor+ position in screen points (origin top-left)."`.++### Intentional additions (Rust-only)++- `structuredContent: { x: int, y: int }` is included alongside the text+ response. Swift returns text only. This is a backwards-compatible MCP+ enrichment — tools that read structured content get integers; tools+ that read text get Swift's exact format. The test asserts both views+ agree and both agree with the platform's native `GetCursorPos` call+ within ±5 px.++### Underlying API per platform++| Platform | Swift | Rust |+|----------|------------------------------------|--------------------------------------------|+| macOS | `CGEvent(source: nil).location` | `CGEvent::new(CGEventSource(HIDSystemState)).location()` |+| Windows | n/a | `GetCursorPos` (Win32) |+| Linux | n/a | `xproto::query_pointer` on the root window |++All three return screen-coordinate space, top-left origin, matching the+documented Swift behavior.++---++## MCP tool: `get_screen_size`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/GetScreenSizeTool.swift:6-46`+ + `libs/cua-driver/Sources/CuaDriverCore/Capture/ScreenInfo.swift`+- Rust:+ - macos=`crates/platform-macos/src/tools/get_screen_size.rs`+ - windows=`crates/platform-windows/src/tools/impl_.rs` (GetScreenSizeTool)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (GetScreenSizeTool)+- Status: VERIFIED+- Test: `crates/platform-windows/examples/get_screen_size_parity.rs`++### Fixed divergences++1. **Missing `scale_factor`** — Swift's whole reason for this tool is to+ expose the backing scale factor (Retina = 2.0). All three Rust+ platforms were dropping it. Now all three return it:+ - macOS: `NSScreen.mainScreen.backingScaleFactor` via `objc2-app-kit`.+ - Windows: `GetDpiForSystem() / 96.0`.+ - Linux: hardcoded `1.0` (X11 has no per-monitor scale; recorded as a+ known limitation rather than silently dropping the field).+2. **Text format** — was `"Screen: W×H"` (no checkmark, no scale, ×+ unicode); now `"✅ Main display: WxH points @ Sx"` matching Swift.+3. **Error case** — macOS now returns `ToolResult::error("No main display+ detected.")` when `NSScreen.mainScreen` is nil, matching Swift's+ `isError: true` response.+4. **Description** — standardized to Swift's wording across all three.+5. **`structuredContent.scale_factor` key** is snake_case to match+ Swift's `ScreenSize.CodingKeys.scaleFactor = "scale_factor"`.++### Verified on Windows++`get_screen_size_parity.exe` round-trips the pipe and asserts:+- Text matches `"✅ Main display: 1680x1050 points @ 1x"` exactly.+- `structuredContent.width / height / scale_factor` agree with the text.+- All three agree with native `GetSystemMetrics` + `GetDpiForSystem` /96.++### Intentional Rust-only++`structuredContent: { width, height, scale_factor }` is included alongside+the text response. Swift returns text only. Same rationale as+`get_cursor_position` — backwards-compatible MCP enrichment, text format+still matches Swift exactly.++---++## MCP tool: `check_permissions`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/CheckPermissionsTool.swift:6-59`+ + `libs/cua-driver/Sources/CuaDriverCore/Permissions/Permissions.swift`+- Rust:+ - macos=`crates/platform-macos/src/tools/check_permissions.rs` (FIXED, pending macOS run)+ - windows=`crates/platform-windows/src/tools/impl_.rs` (CheckPermissionsTool)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (CheckPermissionsTool)+- Status:+ - macos: OPEN (fixed in source; macOS runner needed to verify)+ - windows / linux: INTENTIONAL_DIVERGENCE+- Test: TODO macOS — needs a macOS machine or CI runner to drive the daemon+ and assert the text format + structured response.++### Fixed divergences (macOS)++1. **Schema** — added `prompt: boolean` parameter matching Swift's+ `inputSchema.properties.prompt`.+2. **Default behavior** — Swift defaults `prompt: true` and raises+ the macOS Accessibility + Screen Recording TCC prompts via+ `AXIsProcessTrustedWithOptions({"AXTrustedCheckOptionPrompt": true})`+ and `CGRequestScreenCaptureAccess()`. Rust previously never prompted.+ Both APIs are now wired up via `link(name = "ApplicationServices"|"CoreGraphics")`.+3. **Text format** — was `"Accessibility API: ✅ granted\nScreen Recording: ✅ granted"`;+ now matches Swift exactly: `"✅ Accessibility: granted.\n✅ Screen Recording: granted."`.+4. **Annotation** — `read_only: false` (was `true`) matching Swift's+ `readOnlyHint: false` — the default path can mutate state by raising a dialog.+5. **Description** — copied from Swift verbatim.+6. **Screen Recording probe** — Swift uses `SCShareableContent.excludingDesktopWindows`+ (ScreenCaptureKit), which is hard to call from Rust without large bindings.+ Approximation: `CGPreflightScreenCaptureAccess()` (preflight C API) with+ fallback to the existing window-enumeration heuristic. May report+ false negatives for some subprocess-launched apps (same caveat that+ prompted Swift to switch to SCShareableContent). Documented as a+ known approximation.++### Windows / Linux intentional divergence++`check_permissions` on macOS is fundamentally about TCC (Apple's per-app+permission database). Neither Windows nor Linux have an equivalent. The+Rust ports retain the tool name and the read-only-status structure, but+return platform-specific content:++- **Windows** reports process elevation (admin vs standard) and confirms+ PostMessage + UIA work without elevated rights. The whole premise of+ cua-driver-rs on Windows is that background automation needs no+ special permissions, so the tool's role is to *confirm* that.+- **Linux** reports X11 display reachability, D-Bus session presence,+ and XSendEvent availability — the inputs the X11 backends actually+ need to work.++These divergences are intentional and not fixable without making the tool+no-op on Windows/Linux (worse UX than returning the platform-specific+status).++---++## MCP tool: `list_apps`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/ListAppsTool.swift:6-71`+ + `libs/cua-driver/Sources/CuaDriverCore/Apps/AppInfo.swift`+- Rust:+ - macos=`crates/platform-macos/src/tools/list_apps.rs` + `apps.rs:format_app_list`+ - windows=`crates/platform-windows/src/tools/impl_.rs` (ListAppsTool)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (ListAppsTool)+- Status:+ - macos: code fixed (✅ checkmark added, description copied from Swift); pending macOS run+ - windows: VERIFIED (text format + structured shape + `active` flag)+ - linux: OPEN (subagent currently fixing pre-existing compile errors)+- Test: `crates/platform-windows/examples/list_apps_parity.rs`++### Fixed divergences++1. **Windows: returned every OS process** (System, Registry, csrss, etc.)+ instead of "apps". Now filters to processes that own at least one+ visible top-level window — the closest analogue to Swift's+ `NSApplicationActivationPolicyRegular` filter (excludes background+ helpers / agents).+2. **Structured shape**: Swift returns `Output { apps: [AppInfo] }` with+ `AppInfo = {pid, bundle_id, name, running, active}`. Windows was+ returning `{ processes: [{pid, name}] }`. Now returns both keys —+ `apps` (Swift shape) for new callers, `processes` (legacy shape) as a+ transitional alias.+3. **Text format**: Swift starts with `"✅ Found N app(s): R running, I installed-not-running."`.+ macOS Rust was missing the `✅` prefix; Windows had a different+ wording entirely (`"Found N processes:"`). Both now match Swift 1:1.+4. **`active` flag** — Windows resolves it via `GetForegroundWindow` ++ `GetWindowThreadProcessId` (only one app is `active` at a time).+5. **Description** — copied verbatim from Swift on macOS; adapted with+ Windows-specific caveats on Windows (no bundle_id, no installed-apps+ enumeration yet).++### Known limitation (Windows / Linux)++Neither Rust port enumerates *installed but not running* apps yet — Swift+scans `/Applications`, `~/Applications`, `/System/Applications`, etc.+Windows would need to scan Start Menu shortcuts and the Registry+Uninstall keys; Linux would need to scan `.desktop` files in+`/usr/share/applications` and `~/.local/share/applications`. Marked as+follow-up work; the running-app view is still useful and matches Swift's+running subset exactly.++### Verified on Windows++`list_apps_parity.exe`: header line matches `"✅ Found N app(s): R+running, 0 installed-not-running."`; `structuredContent.apps` is an+array of `{pid, bundle_id (null on Windows), name, running, active}`;+at least one entry has `active: true` (the foreground app).++---++## MCP tool: `list_windows`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift:6-245`+- Rust:+ - macos=`crates/platform-macos/src/tools/list_windows.rs` (TBD audit)+ - windows=`crates/platform-windows/src/tools/impl_.rs` (ListWindowsTool)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (ListWindowsTool, blocked behind Linux compile fix)+- Status:+ - windows: VERIFIED+ - macos: OPEN (audit pending — macOS port already exists)+ - linux: OPEN+- Test: `crates/platform-windows/examples/list_windows_parity.rs`++### Fixed divergences (Windows)++Swift returns per-record: `{window_id, pid, app_name, title, bounds {x,y,width,height}, layer, z_index, is_on_screen, on_current_space?, space_ids?}`+plus top-level `{windows, current_space_id}`. Windows Rust was returning+a flat `{window_id, pid, title, x, y, width, height}`. Now matches Swift:++1. **`app_name`** — populated by joining against the process table+ (`crate::win32::list_processes`).+2. **`bounds: {x, y, width, height}`** — was flat `x`, `y`, `width`,+ `height` siblings of `title`.+3. **`layer: 0`** — Swift filters to layer-0 (normal windows); Windows+ has no layer concept so hard-coded 0.+4. **`z_index`** — derived from `EnumWindows` order (top-to-bottom),+ inverted so higher = closer to front per Swift convention.+5. **`is_on_screen: true`** — currently always true because the Win32+ `list_windows` source filter only returns `IsWindowVisible && !IsIconic`+ windows. See limitation below.+6. **Top-level `current_space_id: null`** — Windows has no Spaces.+7. **`on_current_space` / `space_ids` omitted** — matching Swift's+ else-branch when SkyLight SPIs are unavailable; the text header+ explicitly says so.+8. **Text format** — header now reads `"✅ Found N window(s) across M+ app(s); X on-screen. (SkyLight Space SPIs unavailable — ...)"`.+ Per-record line: `"- {app_name} (pid {pid}) {"title"|(no title)} [window_id: {id}]{[off-screen]?}"`.+9. **Pid-filter warning** — when a pid filter returns zero windows,+ the response is `"⚠️ No windows found for pid X. ..."` with a hint+ about the current frontmost app, matching Swift's warning.+10. **Description** — copied from Swift with Windows-specific caveats+ about Spaces.++### Legacy alias++`structuredContent._legacy_windows` keeps the old flat shape (no `app_name`,+flat `x/y/width/height`) for any pre-existing callers; remove once they migrate.++### Known limitation++Windows Rust does **not** yet enumerate **off-screen / minimized windows**.+Swift's default (`on_screen_only: false`) returns them; Windows currently+filters them out at the `EnumWindows` callback level (`IsWindowVisible &&+!IsIconic`). The `on_screen_only` schema field is accepted but has no+effect. Follow-up to refactor `list_windows` to return everything and+filter at the tool layer.++---++## MCP tool: `click`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift:29-595`+- Rust:+ - macos=`crates/platform-macos/src/tools/click.rs` (TBD audit)+ - windows=`crates/platform-windows/src/tools/impl_.rs` (ClickTool)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (ClickTool)+- Status:+ - windows: VERIFIED (text format + error wording); schema divergences documented below+ - macos: OPEN (already exists; line-by-line audit pending)+ - linux: OPEN+- Test: `crates/platform-windows/examples/click_parity.rs`++### Fixed (Windows)++1. **Text format — pixel path**: was `"✅ Clicked at (X.X, Y.Y) × N."`;+ now `"✅ Posted click to pid X."`, `"✅ Posted double-click to pid X."`,+ `"✅ Posted triple-click to pid X."` matching Swift's `performPixelClick`.+2. **Text format — element path**: was `"✅ Clicked element [N] at screen (X,Y)."`;+ now `"✅ Performed {action} on [N] (screen (X,Y))."` matching Swift's+ `"✅ Performed AXPress on [N] {role} \"{title}\"."` shape (UIA has no+ readily-available role/name on the cached element, so we emit+ element_index + screen coords for traceability; future work: populate+ the UIA cache with name/control-type and include them).+3. **Missing-target error wording** — Swift: `"Provide element_index or+ (x, y) to address the click target."` — Windows previously said+ `"Provide element_index or (x + y). pid is always required."` Now+ matches Swift verbatim.+4. **Description** — multi-paragraph, ported from Swift, with explicit+ notes about Windows-only fields and missing fields.++### Intentional Rust-only schema divergences++Swift's `click` takes `{action: enum, modifier: [string], debug_image_out: string}`;+Windows's `click` takes `{button: enum}` instead. Rationale:++- **`button: left|right|middle`** — Windows convenience. Swift exposes+ right-click as a separate `right_click` tool (which we also have as+ `right_click`), so `button: right` overlaps that tool. Windows+ keeps both shapes; the standalone tool matches Swift's per-tool+ decomposition while `button` gives single-call flexibility.+- **`action: enum`** — AX-specific (AXPress / AXShowMenu / AXPick /+ AXConfirm / AXCancel / AXOpen). UIA has no clean 1:1 mapping (Invoke+ pattern handles most cases; ShowMenu doesn't exist as a UIA pattern).+ Not exposed on Windows yet; a future Windows port could map a subset+ to UIA patterns (Invoke ≈ press, but show_menu has no analogue).+- **`modifier: [string]`** — not yet implemented on Windows; UIA's+ background-click via PostMessage doesn't propagate modifier-key state+ cleanly (would require synthesizing `WM_KEYDOWN(VK_CONTROL)` first).+ Follow-up.+- **`debug_image_out`** — also follow-up.++### Verified on Windows++`click_parity.exe` against Chrome (pid 62156, window_id 4464038):+- Missing-target: `"Provide element_index or (x, y) to address the click target."` ✓+- Pixel click: `"✅ Posted click to pid 62156."` ✓+- Double-click (`count: 2`): `"✅ Posted double-click to pid 62156."` ✓++---++## MCP tool: `launch_app`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift:6-490`+- Rust:+ - windows=`crates/platform-windows/src/tools/impl_.rs` (LaunchAppTool)+ - macos=`crates/platform-macos/src/tools/launch_app.rs` (TBD audit)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (TBD audit)+- Status:+ - windows: VERIFIED+ - macos: OPEN+ - linux: OPEN+- Test: `crates/platform-windows/examples/launch_app_parity.rs`++### Fixed (Windows)++1. **pid capture** — was using `ShellExecuteW` which returns only an+ HINSTANCE that's useless for pid lookup. Now uses+ `ShellExecuteExW` with `SEE_MASK_NOCLOSEPROCESS` so we read the+ spawned process handle and call `GetProcessId` → real pid in the+ response, matching Swift's `AppLauncher.launch.info.pid`.+2. **Structured response** — was missing entirely. Now returns+ `{pid, bundle_id, name, running, active, windows}` matching Swift's+ `LaunchResult` shape exactly. `bundle_id` is null on Windows.+3. **Text format** — was `"Launched 'X' (no focus steal)."`; now+ `"✅ Launched <name> (pid <N>) in background."` + a `Windows:` block+ listing per-window `"- <title|(no title)> [window_id: ID]"` lines+ and a `→ Call get_window_state(...)` hint, matching Swift verbatim.+4. **`bundle_id` parameter** — accepted as an alias for `name` (Windows+ has no bundle-identifier concept). Cross-platform callers can use+ the same field name.+5. **`additional_arguments`** — honored, passed as `lpParameters` to+ `ShellExecuteExW`.+6. **Window-resolution retry** — ports Swift's 5-attempt 100/200ms+ retry to absorb Win32 window-creation lag after `ShellExecuteEx`+ returns.+7. **Error wording** — `"Provide either bundle_id or name to identify+ the app to launch."` matches Swift's `errorResult` text.+8. **`active: false`** — hardcoded; `SW_SHOWNOACTIVATE` is the+ Windows-equivalent of Swift's background-launch invariant.+9. **Description** — multi-paragraph port from Swift with explicit+ Windows-specific notes (path takes precedence; bundle_id alias).++### Intentional Rust-only fields accepted (no-op)++- `electron_debugging_port`, `webkit_inspector_port`,+ `creates_new_application_instance` — Swift-specific. Accepted in+ the schema so cross-platform callers can pass them; currently+ no-ops on Windows. Documented as follow-up.++### Verified on Windows++`launch_app_parity.exe` launches Notepad:+- Header `"✅ Launched notepad.exe (pid 30612) in background."` ✓+- `structuredContent.pid` is the actual ShellExecuteEx pid ✓+- `bundle_id: null`, `running: true`, `active: false` ✓+- Notepad killed on test exit. ✓++---++## MCP tool: `press_key`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/PressKeyTool.swift:20-202`+- Rust:+ - windows=`crates/platform-windows/src/tools/impl_.rs` (PressKeyTool)+ - macos=`crates/platform-macos/src/tools/press_key.rs` (TBD)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (TBD)+- Status: windows VERIFIED; macos / linux OPEN+- Test: `crates/platform-windows/examples/press_key_parity.rs`++### Fixed (Windows)++1. **Text format** — was `"Pressed key 'KEY'."`; now matches Swift verbatim+ `"✅ Pressed KEY on pid X."` (no quotes around key, pid included).+2. **Error wording** —+ - Missing pid: `"Missing required integer field pid."` (was generic)+ - Missing key: `"Missing required string field key."` (was generic)+ - element_index without window_id: `"window_id is required when+ element_index is used — the element_index cache is scoped per+ (pid, window_id). Pass the same window_id you used in+ get_window_state."` (was previously not validated; added the+ same Swift guard).+3. **Schema** — added `element_index` field (accepted; currently no-op+ on Windows since UIA SetFocus isn't wired up yet — documented).+4. **Description** — multi-paragraph port from Swift, including the+ full key vocabulary list.++### Intentional Rust-only (Windows)++- `element_index` is accepted but currently no-op (Swift focuses the+ element via `AXSetAttribute(kAXFocused, true)` first; Windows UIA+ `IUIAutomationElement::SetFocus` exists but is not yet wired up+ here). Documented as a follow-up; using `press_key` without+ `element_index` already works for scroll keys / shortcuts on the+ already-focused element.++### Verified on Windows++`press_key_parity.exe` against Chrome:+- Missing-key error: `"Missing required string field key."` ✓+- Element-without-window error: matches Swift's wording ✓+- End key: `"✅ Pressed end on pid 85676."` ✓++---++## MCP tool: `hotkey`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/HotkeyTool.swift:6-142`+- Rust:+ - windows=`crates/platform-windows/src/tools/impl_.rs` (HotkeyTool)+ - macos=`crates/platform-macos/src/tools/hotkey.rs` (TBD)+ - linux=`crates/platform-linux/src/tools/impl_.rs` (TBD)+- Status: windows VERIFIED; macos / linux OPEN+- Test: `crates/platform-windows/examples/hotkey_parity.rs`++### Fixed (Windows)++1. **Text format** — was `"Pressed CTRL+C on pid X."` (no checkmark,+ uppercase mods only); now `"✅ Pressed K1+K2+... on pid X."`+ matching Swift's `keys.joined(separator: "+")` of the full keys+ array as the caller supplied it (preserves case and order).+2. **Error wording** —+ - Missing pid: schema-layer catches first; tool layer falls back to+ `"Missing required integer field pid."` (Swift's wording).+ - Missing keys: `"Missing required array field keys."` (was+ a Rust-specific "Provide 'keys' array..." fallback).+ - Non-string elements: `"keys must be a non-empty array of strings."`.+3. **Description** — multi-paragraph port from Swift, dropping the+ macOS-specific FocusWithoutRaise / NSMenu mechanics that have no+ Windows analogue.++### Verified on Windows++`hotkey_parity.exe` against Chrome:+- Missing-pid / missing-keys errors ✓+- `ctrl+end` → `"✅ Pressed ctrl+end on pid 85676."` ✓++---++## MCP tool: `double_click`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/DoubleClickTool.swift:28-327`+- Rust:+ - windows=`crates/platform-windows/src/tools/impl_.rs` (DoubleClickTool)+ - macos / linux: OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/double_click_parity.rs`++### Fixed (Windows)++1. **Text format pixel path** — was `"✅ Double-clicked at (X.X, Y.Y)."`;+ now `"✅ Posted double-click to pid X at window-pixel (a, b) → screen-point (c, d)."`+ matching Swift verbatim.+2. **Text format element path** — was `"✅ Double-clicked element [N] at screen (X,Y)."`;+ now `"✅ Posted double-click to [N] at screen-point (X, Y)."` matching+ Swift's pixel-fallback element wording. (UIA role/title placeholder+ pending element-cache enrichment.)+3. **Validation guards added** — ports Swift's full set:+ - `"Provide both x and y together, not just one."`+ - `"Provide either element_index or (x, y), not both."`+ - `"Provide element_index or (x, y) to address the double-click target."`+ - `"window_id is required when element_index is used — the element_index+ cache is scoped per (pid, window_id). Pass the same window_id you used+ in get_window_state."`+4. **Error wording for missing pid** — `"Missing required integer field pid."`+ matches Swift (schema-layer catches this first; tool-layer fallback uses+ Swift wording).+5. **`modifier` schema field** — accepted for parity (no-op on Windows;+ PostMessage doesn't propagate modifier-key state — same caveat as+ `click`).+6. **Multi-paragraph description** ported from Swift with Windows-specific+ notes (no AXOpen analogue; element path always falls back to pixel+ double-click — user-visible behavior matches Swift's AXOpen-not-advertised+ fallback).++### Verified on Windows++`double_click_parity.exe` against Chrome:+- 4 error paths (missing-target, partial-xy, both-modes,+ element-without-window) ✓+- Pixel double-click: `"✅ Posted double-click to pid 85676 at+ window-pixel (300, 300) → screen-point (462, 456)."` ✓++---++## MCP tool: `right_click`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/RightClickTool.swift:27-324`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (RightClickTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/right_click_parity.rs`++### Fixed (Windows)+1. **Pixel-path text** — was `"✅ Right-clicked at (X.X, Y.Y)."`; now+ `"✅ Posted right-click to pid X at window-pixel (a, b) → screen-point (c, d)."`+ matching Swift.+2. **Element-path text** — was `"✅ Right-clicked element [N] at screen (X,Y)."`;+ now `"✅ Shown menu for [N] (screen (X, Y))."` matching Swift's+ AXShowMenu text (UIA role/title placeholder pending element-cache enrichment).+3. **Validation** — ports Swift's 5 guards verbatim (missing pid,+ partial xy, both modes, missing target, element-without-window).+4. **Description** — multi-paragraph port from Swift with Windows-+ specific caveats (no AXShowMenu analogue wired up; element path+ falls through to pixel recipe — matches Swift's+ AXShowMenu-not-advertised fallback).++### Verified on Windows+`right_click_parity.exe` against Chrome:+- 4 error paths ✓+- Pixel right-click: `"✅ Posted right-click to pid 62156 at window-pixel+ (300, 300) → screen-point (301, 368)."` ✓++---++## MCP tool: `screenshot`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/ScreenshotTool.swift:5-170`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (ScreenshotTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/screenshot_parity.rs`++### Fixed (Windows)+1. **Text format** — was `"Screenshot (window): WxH png."`; now matches+ Swift verbatim: `"✅ Window screenshot — WxH png [window_id: ID]"`+ (em-dash, checkmark, window-id suffix). Display fallback uses+ `"✅ Display screenshot — WxH png"` (Rust-only, see intentional below).+2. **Default JPEG quality** — was 85, Swift defaults 95. Now 95.+3. **Description** — multi-paragraph port from Swift adapted to Windows+ (BitBlt + PrintWindow transport; no permission gate needed).+4. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grab+ every call). Now matches Swift.++### Intentional Rust-only++- **Optional `window_id`** — Swift requires it; Windows allows omission+ for whole-display capture (`screenshot_display_bytes`). Useful+ Windows-only convenience that Swift can't easily provide because+ macOS Screen Recording requires per-window grants. Schema accepts+ both shapes; description explains.++### Verified on Windows++`screenshot_parity.exe`:+- Window screenshot: text matches `"✅ Window screenshot — 1087x644 png+ [window_id: 4464038]"` ✓+- Image content block present ✓+- structuredContent has `width`, `height`, `format` ✓+- JPEG format: `mimeType: image/jpeg`, `format: "jpeg"` ✓++---++## MCP tool: `scroll`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/ScrollTool.swift:23-211`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (ScrollTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/scroll_parity.rs`++### Fixed (Windows)+1. **Text format** — was `"Scrolled DIR Nx GRAN."`; now matches Swift+ shape `"✅ Scrolled pid X DIR via Nx SB_LINEDOWN message(s)."` (Swift uses key+ names; Rust uses Win32 SB_* constants since the actual transport is+ WM_VSCROLL/WM_HSCROLL, not keystrokes — text reflects mechanism).+2. **Error wording** — `"Missing required integer field pid."` /+ `"Missing required string field direction."` matches Swift.+3. **Element-without-window guard** added — same wording as Swift.+4. **Description** — multi-paragraph port from Swift with Windows-+ specific note about WM_VSCROLL/WM_HSCROLL vs Swift's keystroke+ approach.+5. **`element_index`** — accepted; currently no-op on Windows (UIA+ SetFocus not wired up).++### Intentional Rust-only++- **Transport**: Windows uses `WM_VSCROLL`/`WM_HSCROLL` messages with+ `SB_LINE*`/`SB_PAGE*` codes. Swift uses synthesized keystrokes+ (PageDown/arrow keys) via auth-signed SLEventPostToPid because+ CGEventCreateScrollWheelEvent2 is dropped by Chromium on macOS.+ Windows doesn't have that constraint; scroll-message events work+ reliably background.++### Verified on Windows++`scroll_parity.exe` against Chrome:+- Missing-direction error ✓+- Element-without-window error ✓+- Page-down × 2: `"✅ Scrolled pid 62156 down via 2× SB_PAGEDOWN message(s)."` ✓++---++## MCP tool: `type_text`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swift:13-225`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (TypeTextTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/type_text_parity.rs`++### Fixed (Windows)+1. **Text format** — was `"Typed N character(s)."`; now matches Swift's+ CGEvent-fallback shape `"✅ Typed N char(s) on pid X via PostMessage (Yms delay)."`+ (Swift says `CGEvent`; Windows says `PostMessage` since that's the+ actual transport).+2. **Error wording** — `"Missing required integer field pid."` /+ `"Missing required string field text."` matches Swift.+3. **Element-without-window guard** added with Swift's wording.+4. **`delay_ms` field** added (0–200, default 30, matches Swift).+5. **Description** — multi-paragraph port from Swift adapted to the+ PostMessage transport.++### Intentional Rust-only++- **No AX-fast-path attempt**: Swift tries `AXSetAttribute(kAXSelectedText)`+ first for bulk insert, falls back to CGEvent when AX rejects. Windows+ always takes the character-by-character path (no UIA equivalent of+ AXSelectedText wired up yet). User-visible behavior matches Swift's+ fallback path, just without the fast-path optimization.+- **`element_index`** accepted; no-op on Windows (no UIA SetFocus path).++### Verified on Windows+`type_text_parity.exe` against Chrome:+- Missing-text error ✓+- Element-without-window error ✓+- 5-char type: `"✅ Typed 5 char(s) on pid 62156 via PostMessage (10ms delay)."` ✓++---++## MCP tool: `set_value`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift:8-336`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (SetValueTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/set_value_parity.rs`++### Fixed (Windows)+1. **Text format** — was `"Set value of element N."`; now matches Swift's+ default-write shape `"✅ Set AXValue on [N] (UIA ValuePattern)."` (UIA+ role/title placeholder pending element-cache enrichment).+2. **`idempotent: true`** — was `false`; Swift uses `true` (setting same+ value twice is idempotent).+3. **Error wording** — Swift's "Missing required integer fields pid,+ window_id, and element_index." for the all-ints-missing case;+ schema-layer wording for individual missing fields.+4. **Description** — multi-paragraph port from Swift adapted to UIA+ ValuePattern transport, with note about Swift's AXPopUpButton+ special-case (UIA SelectionPattern analogue not yet wired up).++### Intentional Rust-only++- **No AXPopUpButton special-case** yet — Swift specifically iterates+ AX children to AXPress the matching option (bypassing the native+ popup menu). UIA has `IUIAutomationSelectionPattern` /+ `IUIAutomationSelectionItemPattern` which would be the equivalent;+ documented as a follow-up. For now `set_value` on a combo box+ delegates to `IUIAutomationValuePattern::SetValue` which works on+ most native ComboBoxes.++### Verified on Windows+`set_value_parity.exe`:+- Missing-value error (schema layer): mentions `value` + `required` ✓+- Cache-miss error: `"Element 99999 not in cache."` ✓++---++## MCP tools: `get_config` + `set_config`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/GetConfigTool.swift:13-79`+ + `libs/cua-driver/Sources/CuaDriverServer/Tools/SetConfigTool.swift:25-167`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (GetConfigTool, SetConfigTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/config_parity.rs`++### Fixed (Windows)+1. **get_config text** — was `"cua-driver-rs configuration"`; now matches+ Swift's `"✅ <pretty JSON>"` format with `schema_version`, `version`,+ `platform`, `capture_mode`, `max_image_dimension`.+2. **set_config schema** — now accepts Swift's `{key: <dotted-path>, value: <json>}`+ shape AND keeps the legacy per-field shape. Unknown keys return+ `"Unknown config key 'X'. Known: capture_mode, max_image_dimension."`.+3. **set_config response** — was `"Config updated: ..."`; now echoes the+ full updated config in the same pretty-JSON `"✅ {...}"` format as+ `get_config`, matching Swift's "echo full config after write" pattern.+4. **Descriptions** — both ported from Swift with Windows-specific+ schema notes.++### Intentional Rust-only++- **No `agent_cursor.*` subtree** in the Windows config struct. Swift+ exposes `agent_cursor.enabled` + `agent_cursor.motion.*` (start_handle,+ end_handle, arc_size, arc_flow, spring) as persistent config; Windows+ currently has only `capture_mode` and `max_image_dimension`. Cursor+ config lives separately in `cursor-overlay` crate config and is set+ via CLI flags, not this tool yet. Documented as a follow-up.++### Verified on Windows+`config_parity.exe`:+- get_config: `"✅ {<pretty JSON>}"` with all keys ✓+- set_config {key, value}: capture_mode=vision applied ✓+- set_config legacy: capture_mode=som + max_image_dimension=1024 applied ✓+- Unknown key error ✓+- Missing-input error ✓++---++## MCP tool: `get_agent_cursor_state`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/GetAgentCursorStateTool.swift:9-68`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (GetAgentCursorStateTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/get_agent_cursor_state_parity.rs`++### Fixed (Windows)+1. **Text format** — was `"N cursor instance(s)."`; now matches Swift's+ single-line camelCase output:+ `"✅ cursor: enabled=true startHandle=0.3 endHandle=0.3 arcSize=0.25+ arcFlow=0 spring=0.72 glideDurationMs=0 dwellAfterClickMs=80 idleHideMs=20000"`+2. **`current_motion()`** helper added to `overlay.rs` — mirrors macOS+ `current_motion()` so the tool can snapshot live motion values.+3. **Description** — ported from Swift verbatim.++### Intentional Rust-only++- **Multi-cursor `cursors` array** — Rust supports multiple cursor+ instances via `CursorRegistry`; Swift has a single `AgentCursor.shared`.+ Included in `structuredContent` as an extra field; text format keeps+ Swift's single-cursor vocabulary.++### Verified on Windows+`get_agent_cursor_state_parity.exe`:+- Text: `"✅ cursor: enabled=true startHandle=0.3 endHandle=0.3 arcSize=0.25+ arcFlow=0 spring=0.72 glideDurationMs=0 dwellAfterClickMs=80 idleHideMs=20000"` ✓+- structuredContent has all 9 fields + `cursors` array ✓++---++## MCP tools: `set_agent_cursor_enabled` + `set_agent_cursor_motion`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/SetAgentCursorEnabledTool.swift:8-85`+ + `libs/cua-driver/Sources/CuaDriverServer/Tools/SetAgentCursorMotionTool.swift:11-187`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs`; macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/agent_cursor_setters_parity.rs`++### Fixed (Windows)+1. **set_agent_cursor_enabled text** — was `"Agent cursor 'default' enabled."`;+ now matches Swift verbatim `"✅ Agent cursor enabled."` (or `"disabled"`).+2. **set_agent_cursor_enabled error** — was `"Missing required parameter: enabled"`;+ now Swift's `"Missing required boolean field \`enabled\`."`.+3. **set_agent_cursor_motion was silently dropping all motion knobs** —+ tool accepted them in schema but only forwarded appearance fields to+ the cursor registry. Now applies each motion knob via+ `MotionConfig::with_overrides()` and sends `OverlayCommand::SetMotion`+ to the live render state — matches Swift's+ `AgentCursor.shared.defaultMotionOptions = opts`.+4. **set_agent_cursor_motion text** — was `"Cursor 'default' config updated."`;+ now matches Swift's `"✅ cursor motion: startHandle=X endHandle=Y arcSize=Z arcFlow=W spring=S glideDurationMs=N dwellAfterClickMs=N idleHideMs=N"`.+5. **Number coercion** — JSON ints (`{"glide_duration_ms": 500}`) now+ coerced to f64 instead of silently ignored (mirrors Swift's `number()`).+6. **Descriptions** — both ported from Swift; appearance/motion split+ documented (Rust splits appearance into the separate+ `set_agent_cursor_style` tool, matching Swift's+ SetAgentCursorStyleTool surface).++### Intentional Rust-only+- **`cursor_id`** parameter for both tools — selects an instance from+ the Rust-only multi-cursor registry. Swift has a single+ `AgentCursor.shared`.++### Verified on Windows+`agent_cursor_setters_parity.exe`:+- Missing-enabled error ✓+- `enabled: true` → `"✅ Agent cursor enabled."` ✓+- `enabled: false` → `"✅ Agent cursor disabled."` ✓+- Motion update: `"✅ cursor motion: startHandle=0.4 endHandle=0.3 arcSize=0.3+ arcFlow=0 spring=0.8 glideDurationMs=500 dwellAfterClickMs=80 idleHideMs=20000"` ✓++---++## MCP tools: `get_recording_state` + `set_recording`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/GetRecordingStateTool.swift:11-72`+ + `libs/cua-driver/Sources/CuaDriverServer/Tools/SetRecordingTool.swift:12-160`+- Rust: shared `crates/mcp-server/src/recording_tools.rs` (cross-platform)+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/recording_parity.rs`++### Fixed+1. **get_recording_state text** — was `"recording: enabled output_dir=X next_turn=N"`+ (double-space-separated); now Swift's `"✅ recording: enabled output_dir=X next_turn=N"`+ single-space. Disabled case: `"✅ recording: disabled"`.+2. **set_recording text** — was `"Recording enabled → X"` (Unicode arrow);+ now Swift's `"✅ Recording enabled -> X"` (ASCII arrow). Disabled:+ `"✅ Recording disabled."`.+3. **Error wording** — Swift's `"Missing required boolean field \`enabled\`."`+ and `"`output_dir` is required when enabling recording."`.+4. **`idempotent: true`** — was `false`; Swift uses `true`.+5. **Descriptions** — both ported from Swift verbatim with full turn-folder+ layout details.++### Verified on Windows+`recording_parity.exe`:+- `"✅ recording: disabled"` ✓+- Missing-enabled error ✓+- Missing-output_dir error ✓+- Enable: `"✅ Recording enabled -> <path>"` ✓+- `"✅ recording: enabled output_dir=<path> next_turn=1"` ✓+- Disable: `"✅ Recording disabled."` ✓++---++## CLI subcommand: `list-tools`+- Swift: `libs/cua-driver/Sources/CuaDriverCLI/CallCommand.swift:298-322`+- Rust: `libs/cua-driver-rs/crates/cua-driver/src/cli.rs::run_list_tools`+- Status: VERIFIED+- Test: `crates/platform-windows/examples/list_tools_parity.rs`++### Fixed+**Sort order** — Swift sorts tools alphabetically by name+(`tools.sorted(by: { $0.name < $1.name })`). Rust was iterating in+registration order (HashMap-backed Vec). Now sorts alphabetically+before printing, matching Swift's output byte-for-byte modulo per-tool+description content.++### Verified on Windows+`list_tools_parity.exe`:+- Names sorted ascending ✓+- All 23 core tools present (click, double_click, right_click, type_text,+ press_key, hotkey, scroll, screenshot, list_apps, list_windows,+ get_cursor_position, get_screen_size, launch_app, move_cursor,+ set_agent_cursor_enabled, set_agent_cursor_motion,+ get_agent_cursor_state, set_value, get_config, set_config,+ check_permissions, get_recording_state, set_recording) ✓+- Line shape `name: <first sentence>` or just `name` ✓++---++## CLI subcommand: `describe`+- Swift: `libs/cua-driver/Sources/CuaDriverCLI/CallCommand.swift:324-366`+ + `printUnknownTool:552`+- Rust: `libs/cua-driver-rs/crates/cua-driver/src/cli.rs::run_describe`+- Status: VERIFIED+- Test: `crates/platform-windows/examples/describe_parity.rs`++### Fixed+**Unknown-tool listing order** — Swift sorts available tools+alphabetically in the error stderr block. Rust was using+`tool_names()` which is registration order. Now sorts to match.+Exit code 64 (EX_USAGE) and known-tool output shape (`name: X`++ `description:` + `input_schema:` pretty JSON) already matched+Swift.++### Verified on Windows+`describe_parity.exe`:+- `describe click`: stdout starts with `"name: click\n"`, contains+ `"description:"` + pretty-printed `"input_schema:"` ✓+- `describe this_tool_does_not_exist`: exit code 64, stderr lists+ 31 tools alphabetically with `"Unknown tool:"` + `"Available tools:"`+ preamble ✓++---++## MCP tool: `get_window_state`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swift:5-end`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (GetWindowStateTool); macos/linux OPEN+- Status: windows VERIFIED (error wording + validation); response shape already verified+- Test: `crates/platform-windows/examples/get_window_state_parity.rs`++### Fixed (Windows)+1. **Error wording** — Swift's verbatim messages now used:+ - `"Missing required integer field pid."`+ - `"Missing required integer field window_id. Use `list_windows` to enumerate the target app's windows, or read `launch_app`'s `windows` array."`+2. **Window-belongs-to-pid validation** — Swift hard-errors when the+ window_id doesn't belong to pid. Windows now validates the same way:+ - Window doesn't exist anywhere: `"No window with window_id N exists.+ Call list_windows({pid: P}) for candidates."`+ - Window exists under a different pid: `"window_id N belongs to pid Q,+ not pid P. Call list_windows({pid: P}) to get this pid's own windows."`+3. **`idempotent: false`** — was `true`; Swift uses `false` (each call+ is a fresh snapshot).+4. **Description** — multi-paragraph port from Swift covering invariant,+ `capture_mode` semantics, `query` filter behavior, and Windows-specific+ notes (UIA instead of AX, no Spaces, no `javascript` / `screenshot_out_file`+ fields).++### Intentional Rust-only schema absences+- **`javascript`** — macOS-only AppleScript hook for Chromium/Safari.+- **`screenshot_out_file`** — could be added later; not currently+ implemented on Windows.++### Verified on Windows+`get_window_state_parity.exe`:+- Missing-pid error ✓+- Missing-window_id error (with Swift's full hint) ✓+- Mismatched pid+window_id: `"window_id N belongs to pid Q, not pid P..."` ✓+- Bogus window_id: `"No window with window_id N exists..."` ✓++---++## MCP tool: `drag`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/DragTool.swift:21-327`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (DragTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/drag_parity.rs`++### Fixed (Windows)+1. **Text format** — was `"✅ Posted drag (BUTTON) to pid P from (x,y) → (x,y) in Nms / Ssteps."`;+ now Swift's `"✅ Posted drag[ (BUTTON button)] to pid P from window-pixel (a, b) → (c, d), screen (e, f) → (g, h) in Nms / Ssteps."`+ (window-pixel + screen-coord pair; button suffix only for non-left).+2. **Missing-coordinates error** — was `"Missing: from_y"` (one field at a+ time); now Swift's `"from_x, from_y, to_x, and to_y are all required+ (window-local pixels)."` even if only one is missing.+3. **Missing-pid error** — `"Missing required integer field pid."`+ matches Swift.++### Intentional Rust-only+- Schema unchanged; all Swift fields supported. No divergences in shape.++### Verified on Windows+`drag_parity.exe`:+- Missing-coordinates error ✓+- Real drag: `"✅ Posted drag to pid 62156 from window-pixel (100, 100) → (102, 102), screen (101, 168) → (103, 170) in 50ms / 2 steps."` ✓++---++## MCP tool: `replay_trajectory`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/ReplayTrajectoryTool.swift:18-end`+- Rust: shared `crates/mcp-server/src/recording_tools.rs` (cross-platform)+- Status: VERIFIED+- Test: `crates/platform-windows/examples/replay_trajectory_parity.rs`++### Fixed+1. **Error wording** — was `"Missing required parameter: dir"`; now Swift's+ `"Missing required string field \`dir\`."`. Empty-string `dir` now also+ rejected (was silently passing the empty path through).+2. **`open_world: false`** — was `true`; Swift uses `false` (replays only+ recorded actions, no fresh world interactions).+3. **Description** — multi-paragraph port from Swift covering caveats+ (element-index actions fail on replay; read-only tools not recorded;+ recording-during-replay deliberate).++### Verified on Windows+`replay_trajectory_parity.exe`:+- Empty-dir: `"Missing required string field \`dir\`."` ✓+- Nonexistent dir: `"Trajectory directory does not exist: <path>"` ✓++---++## CLI subcommands: `status` + `stop`+- Swift: `libs/cua-driver/Sources/CuaDriverCLI/ServeCommand.swift:368-470`+- Rust: `libs/cua-driver-rs/crates/cua-driver/src/serve.rs::run_status_cmd, run_stop_cmd`+- Status: VERIFIED+- Test: `crates/platform-windows/examples/daemon_lifecycle_parity.rs`++### Fixed+**`stop` silent on success** — Rust was printing `"cua-driver daemon stopped."`+on stdout after a successful stop. Swift's `stop` exits silently with+status 0. Now matches Swift byte-for-byte.++### Already correct+- `status` output: `"cua-driver daemon is running\n socket: <path>\n pid: <N>\n"` ✓+- `status` exit code: 0 when running, 1 when not ✓+- `stop` exit code: 0 when ran, 1 when no daemon ✓+- Error wording on stderr: `"cua-driver daemon is not running"` ✓++### Verified on Windows+`daemon_lifecycle_parity.exe`:+- `status` running: 3-line output ✓+- `stop` running: silent stdout (was printing extra line) ✓+- `status` not-running: exit 1 + stderr message ✓+- `stop` not-running: exit 1 + stderr message ✓++---++## MCP tool alias: `type_text_chars` → `type_text`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift:55-70`+- Rust: `libs/cua-driver-rs/crates/cua-driver/src/serve.rs` (both pipe variants)+ + `libs/cua-driver-rs/crates/mcp-server/src/tool.rs::ToolRegistry::invoke`+- Status: VERIFIED+- Test: `crates/platform-windows/examples/type_text_chars_alias_parity.rs`++### Fixed+Swift treats `type_text_chars` as a **deprecated alias** for `type_text`:+the aliased name is NOT in `tools/list`, but invoking it with the old name+works (resolves to `type_text`) AND emits a stderr deprecation warning.+Rust was previously registering `type_text_chars` as a fully-fledged+separate tool with its own description and a different text format.++Changes:+- Remove `TypeTextCharsTool` from the registration in+ `platform-windows/src/tools/impl_.rs::build_registry` (the struct is+ kept in the crate for now via a no-op binding to avoid the dead-code+ warning during incremental cleanup).+- `serve.rs`: both `"call"` dispatch sites (Unix-socket + Windows-pipe+ variants) now resolve `type_text_chars` → `type_text` before the+ registry lookup, with the stderr deprecation message.+- `mcp-server/src/tool.rs::ToolRegistry::invoke`: also resolves the+ alias as a defense-in-depth for direct in-process callers.++### Verified on Windows+`type_text_chars_alias_parity.exe`:+- `tools/list` does NOT contain `"type_text_chars"` ✓+- Invoking `type_text_chars` with a 1-char text resolves to `type_text`'s+ response: `"✅ Typed 1 char(s) on pid 62156 via PostMessage (30ms delay)."` ✓++---++## MCP tool: `set_agent_cursor_style`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/SetAgentCursorStyleTool.swift:10-111`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (SetAgentCursorStyleTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/set_agent_cursor_style_parity.rs`++### Fixed (Windows)+1. **Text format** — was `"cursor style: gradient_colors=[X, Y] bloom_color=Z image_path=(unchanged)"`+ (always lists all three fields with `(unchanged)`/`(reverted)` placeholders,+ space after comma). Now Swift's exact wording: `"✅ cursor style: gradient_colors=[X,Y] bloom_color=Z"`+ (omit fields not provided; no space after comma; `image_path` only when set).+2. **Revert-all case** — was `"cursor style: gradient_colors=(unchanged) bloom_color=(unchanged) image_path=(unchanged)"`;+ now `"✅ cursor style: reverted to default"` matching Swift's `parts.isEmpty` branch.++### Verified on Windows+`set_agent_cursor_style_parity.exe`:+- Set gradient + bloom: `"✅ cursor style: gradient_colors=[#FF6B6B,#FF8E53] bloom_color=#A855F7"` ✓+- Revert all: `"✅ cursor style: reverted to default"` ✓++---++## MCP tool: `zoom`+- Swift: `libs/cua-driver/Sources/CuaDriverServer/Tools/ZoomTool.swift:12-end`+- Rust: windows=`crates/platform-windows/src/tools/impl_.rs` (ZoomTool); macos/linux OPEN+- Status: windows VERIFIED+- Test: `crates/platform-windows/examples/zoom_parity.rs`++### Fixed (Windows)+1. **Text format** — was `"Zoom (X,Y)–(X,Y) → WxH px JPEG."`; now Swift's+ verbatim multi-line message starting `"✅ Zoomed region captured at+ native resolution."` with the `from_zoom=true` integration hint.+2. **Error wording**: `"Missing required integer field pid."` /+ `"Missing required integer field window_id."` /+ `"Missing required region coordinates (x1, y1, x2, y2)."` /+ `"Invalid region: x2 must be > x1 and y2 must be > y1."` /+ `"Zoom region too wide: N px > 500 px max. ..."` — all match Swift.+3. **Schema** — `pid` now also required (Swift requires it).+4. **`idempotent: false`** — was `true`; Swift uses `false`.++### Intentional Rust-only+- **`window_id` required** — Swift uses pid+frontmost-window; Windows+ uses HWND directly since there's no clean Win32 analogue without an+ explicit HWND.++### Verified on Windows+`zoom_parity.exe`: invalid-region, too-wide, and real-zoom paths all OK.++---++## CLI subcommand: `mcp-config`+- Swift: `libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift:37-150`+- Rust: `libs/cua-driver-rs/crates/cua-driver/src/cli.rs::run_mcp_config`+- Status: VERIFIED (no fixes needed — already matched Swift)+- Test: `crates/platform-windows/examples/mcp_config_parity.rs`++### Already correct+- Default (no `--client`): generic mcpServers JSON snippet ✓+- `--client claude`: `claude mcp add --transport stdio cua-driver -- <bin> mcp` ✓+- `--client codex`: `codex mcp add cua-driver -- <bin> mcp` ✓+- `--client cursor`: JSON with `type: stdio` ✓+- `--client openclaw`: `openclaw mcp set ...` ✓+- `--client opencode`: opencode.json snippet with type=local ✓+- `--client hermes`: YAML snippet ✓+- `--client pi`: shell-tool fallback message ✓+- Unknown client: exit 2 + stderr message ✓++### Verified on Windows+`mcp_config_parity.exe` runs all 8 client variants + the unknown-client+error path against the in-process binary, asserting each output contains+the right needles. All pass on first try.++---++## CLI subcommand: `update`+- Swift: `libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift:638-686`+- Rust: `libs/cua-driver-rs/crates/cua-driver/src/cli.rs::run_update_cmd`+- Status: VERIFIED (bug fix)+- Test: `crates/platform-windows/examples/update_parity.rs`++### Fixed — REAL BUG+Rust was looking for release tag prefix `cua-driver-v` (Swift's prefix)+when fetching the latest version from `trycua/cua`. That would match the+Swift `cua-driver-v0.1.9` release and report it as an available upgrade+for the Rust port — confusing users into installing the WRONG binary.++Now uses prefix `cua-driver-rs-v` (Rust port's actual tag prefix).++### Verified on Windows+`update_parity.exe`:+- Always-present lines: `Current version:` and `Checking for updates…` ✓+- Outcome is one of: `Already up to date.`, `New version available: <v>`,+ or `Could not reach GitHub` ✓+- After the fix, the Swift `cua-driver-v0.1.9` tag is no longer mis-+ matched as a Rust-port release.++---++## CLI subcommand: `dump-docs`+- Swift: `libs/cua-driver/Sources/CuaDriverCLI/DumpDocsCommand.swift`+- Rust: `libs/cua-driver-rs/crates/cua-driver/src/cli.rs::run_dump_docs_with_type`+- Status: VERIFIED (with caveat about CLI extraction)+- Test: `crates/platform-windows/examples/dump_docs_parity.rs`++### Fixed+1. **Top-level JSON shape** — was `{mcp_tools: [...]}`; now matches Swift's+ `CombinedDocs` shape: `--type=all` → `{cli: {...}, mcp: {...}}`,+ `--type=mcp` → `{version, tools: [...]}`, `--type=cli` → CLI section.+2. **`--type` flag** added (`cli` | `mcp` | `all`, default `all`) matching+ Swift's flag.+3. **`version` field** added to the MCP section matching Swift's+ `MCPDocumentation`.++### Intentional Rust-only+- **CLI section is a stub** — Swift extracts via swift-argument-parser+ introspection; Rust uses hand-rolled arg matching in `cli.rs` so there's+ no equivalent introspection. Stub returns `{version, commands: [],+ _note: "..."}` directing users to `--help` for CLI docs. Full CLI+ introspection would require either clap migration or a parallel+ hand-maintained doc table.+- **Extra MCP fields** — `read_only`, `destructive`, `idempotent` per+ tool. Swift only emits `name`, `description`, `input_schema`. Rust keeps+ the extras as a documented enrichment.++### Verified on Windows+`dump_docs_parity.exe`:+- Default (`--type=all`): `{cli, mcp}` with 30 tools ✓+- `--type=mcp`: top-level `{version, tools}` ✓+- `--type=cli`: stub section ✓+- `--pretty`: multi-line JSON (991 lines) ✓diff --git a/libs/cua-driver-rs/crates/cua-driver/Cargo.toml b/libs/cua-driver-rs/crates/cua-driver/Cargo.tomlnew file mode 100644index 0000000000..a7c31f469d--- /dev/null+++ b/libs/cua-driver-rs/crates/cua-driver/Cargo.toml@@ -0,0 +1,39 @@+[package]+name = "cua-driver"+version.workspace = true+edition.workspace = true+description = "Cross-platform background computer-use automation daemon (MCP server)"+default-run = "cua-driver"++[[bin]]+name = "cua-driver"+path = "src/main.rs"++[dependencies]+tokio = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+anyhow = { workspace = true }+tracing = { workspace = true }+tracing-subscriber = { workspace = true }+mcp-server = { path = "../mcp-server" }+cursor-overlay = { path = "../cursor-overlay" }+async-trait = "0.1"+base64 = { workspace = true }++[target.'cfg(target_os = "macos")'.dependencies]+platform-macos = { path = "../platform-macos" }++[target.'cfg(target_os = "windows")'.dependencies]+platform-windows = { path = "../platform-windows" }++[target.'cfg(target_os = "linux")'.dependencies]+platform-linux = { path = "../platform-linux" }++[dev-dependencies]+tokio = { workspace = true, features = ["full"] }+image = { workspace = true }+base64 = { workspace = true }++[target.'cfg(target_os = "windows")'.dev-dependencies]+platform-windows = { path = "../platform-windows" }diff --git a/libs/cua-driver-rs/crates/cua-driver/src/cli.rs b/libs/cua-driver-rs/crates/cua-driver/src/cli.rsnew file mode 100644index 0000000000..4215bdfff3--- /dev/null+++ b/libs/cua-driver-rs/crates/cua-driver/src/cli.rs@@ -0,0 +1,1283 @@+//! CLI subcommand parsing and execution.+//!+//! Subcommand dispatch (mirrors the Swift cua-driver CLI):+//!+//! cua-driver → mcp server (default)+//! cua-driver mcp → mcp server (explicit)+//! cua-driver list-tools → print all tool names + descriptions+//! cua-driver describe <tool> → print tool schema+//! cua-driver call <tool> [json-args] → invoke tool, print result+//! cua-driver <tool> [json-args] → shorthand for call (snake_case names)+//!+//! Cursor-overlay flags (--cursor-id, --no-overlay, etc.) are consumed by+//! `CursorConfig::from_args()` and are ignored here.++use std::process;+use mcp_server::{protocol::Content, tool::ToolRegistry};++/// Which CLI command was requested.+pub enum Command {+ Mcp,+ ListTools,+ Describe(String),+ Call { tool: String, json_args: Option<serde_json::Value>, screenshot_out_file: Option<String> },+ McpConfig { client: Option<String> },+ Serve { socket: Option<String> },+ Stop { socket: Option<String> },+ Status { socket: Option<String> },+ Recording { subcommand: String, args: Vec<String>, socket: Option<String> },+ DumpDocs { pretty: bool, doc_type: String },+ Update { apply: bool },+ Doctor,+ Diagnose,+ Config {+ /// `show` | `get` | `set` | `reset` (None → show)+ subcommand: Option<String>,+ /// key arg for `get`/`set`+ key: Option<String>,+ /// value arg for `set`+ value: Option<String>,+ socket: Option<String>,+ },+}++/// Flags whose next token is a value (not a subcommand).+/// We skip both the flag and its value when scanning for the subcommand.+const VALUE_FLAGS: &[&str] = &[+ "--cursor-icon", "--cursor-id", "--cursor-palette",+ "--glide-ms", "--dwell-ms", "--idle-hide-ms",+ "--screenshot-out-file", "--client", "--socket", "--pid-file", "--type",+];++/// Parse the first non-flag positional argument from argv to determine which+/// subcommand to run. Cursor-overlay flags are consumed by `CursorConfig`+/// independently; we only care about the first non-`--` arg here.+pub fn parse_command() -> Command {+ let args: Vec<String> = std::env::args().skip(1).collect();++ // Handle --version / -V before any other parsing so they are never+ // silently stripped as "bare flags" and swallowed by MCP mode.+ if args.iter().any(|a| a == "--version" || a == "-V") {+ println!("cua-driver {}", env!("CARGO_PKG_VERSION"));+ std::process::exit(0);+ }+ if args.iter().any(|a| a == "--help" || a == "-h") {+ println!("cua-driver {} — cross-platform computer-use automation driver", env!("CARGO_PKG_VERSION"));+ println!("Usage: cua-driver [SUBCOMMAND] [OPTIONS]");+ println!("Subcommands: mcp, list-tools, describe, call, serve, stop, status, config, recording, update, doctor, diagnose");+ std::process::exit(0);+ }++ // Collect named flag values we care about.+ let screenshot_out_file = flag_value(&args, "--screenshot-out-file");+ let mcp_client = flag_value(&args, "--client");+ let socket = flag_value(&args, "--socket");++ // Strip cursor-overlay flags (and their values) to expose the subcommand.+ let mut positionals: Vec<&str> = Vec::new();+ let mut i = 0;+ while i < args.len() {+ let a = args[i].as_str();+ if VALUE_FLAGS.contains(&a) {+ i += 2; // skip flag + value+ } else if a.starts_with('-') {+ i += 1; // skip bare flag+ } else {+ positionals.push(a);+ i += 1;+ }+ }++ let mut pos = positionals.into_iter();+ match pos.next() {+ None | Some("mcp") => Command::Mcp,+ Some("list-tools") => Command::ListTools,+ Some("mcp-config") => Command::McpConfig { client: mcp_client },+ Some("serve") => Command::Serve { socket },+ Some("stop") => Command::Stop { socket },+ Some("status") => Command::Status { socket },+ Some("recording") => {+ let subcommand = pos.next().unwrap_or("status").to_string();+ let rest: Vec<String> = pos.map(str::to_owned).collect();+ Command::Recording { subcommand, args: rest, socket }+ }+ Some("dump-docs") => {+ let pretty = args.iter().any(|a| a == "--pretty" || a == "-p");+ let doc_type = flag_value(&args, "--type").unwrap_or_else(|| "all".to_owned());+ Command::DumpDocs { pretty, doc_type }+ }+ Some("update") => {+ let apply = args.iter().any(|a| a == "--apply");+ Command::Update { apply }+ }+ Some("doctor") => Command::Doctor,+ Some("diagnose") => Command::Diagnose,+ Some("config") => {+ let subcommand = pos.next().map(str::to_owned);+ let key = pos.next().map(str::to_owned);+ let value = pos.next().map(str::to_owned);+ Command::Config { subcommand, key, value, socket }+ }+ Some("describe") => {+ let name = pos.next().unwrap_or("").to_string();+ Command::Describe(name)+ }+ Some("call") => {+ let tool = pos.next().unwrap_or("").to_string();+ let json_args = pos.next()+ .and_then(|s| serde_json::from_str(s).ok())+ .or_else(|| read_stdin_json());+ Command::Call { tool, json_args, screenshot_out_file }+ }+ Some(first) => {+ // Implicit call: unrecognised first positional → treat as tool name.+ let tool = first.to_string();+ let json_args = pos.next()+ .and_then(|s| serde_json::from_str(s).ok())+ .or_else(|| read_stdin_json());+ Command::Call { tool, json_args, screenshot_out_file }+ }+ }+}++/// Return the value of `--flag value` from argv, or `None`.+fn flag_value(args: &[String], flag: &str) -> Option<String> {+ let mut it = args.iter();+ while let Some(a) = it.next() {+ if a == flag {+ return it.next().cloned();+ }+ // Also handle --flag=value form.+ if let Some(rest) = a.strip_prefix(flag) {+ if let Some(val) = rest.strip_prefix('=') {+ return Some(val.to_owned());+ }+ }+ }+ None+}++/// Print all tools in the registry, one per line: `name: first sentence`.+pub fn run_list_tools(registry: &ToolRegistry) {+ // Sort alphabetically by name to match Swift's+ // `ListToolsCommand.run()` `tools.sorted(by: { $0.name < $1.name })`.+ let mut entries: Vec<(&str, &mcp_server::tool::ToolDef)> = registry.iter_defs().collect();+ entries.sort_by(|a, b| a.0.cmp(b.0));+ for (name, def) in entries {+ let summary = first_sentence(&def.description);+ if summary.is_empty() {+ println!("{name}");+ } else {+ println!("{name}: {summary}");+ }+ }+}++/// Print a tool's full description and JSON input schema.+/// Exits 64 (EX_USAGE) if the tool is unknown.+pub fn run_describe(registry: &ToolRegistry, name: &str) {+ match registry.get_def(name) {+ None => {+ eprintln!("Unknown tool: {name}");+ eprintln!("Available tools:");+ // Sort alphabetically to match Swift's `printUnknownTool`+ // (`registry.allTools.map(\.name).sorted()`).+ let mut names: Vec<&str> = registry.tool_names().collect();+ names.sort();+ for n in names {+ eprintln!(" {n}");+ }+ process::exit(64);+ }+ Some(def) => {+ print!("name: {}\n", def.name);+ if !def.description.is_empty() {+ print!("\ndescription:\n{}", def.description);+ if !def.description.ends_with('\n') { println!(); }+ }+ print!("\ninput_schema:\n");+ let pretty = serde_json::to_string_pretty(&def.input_schema)+ .unwrap_or_else(|_| def.input_schema.to_string());+ println!("{pretty}");+ }+ }+}++/// Print the MCP server config snippet or a client-specific install command.+///+/// `--client <name>` selects one of: claude, codex, cursor, hermes, openclaw,+/// opencode. Omit for the generic JSON snippet.+pub fn run_mcp_config(client: Option<&str>) {+ let binary = std::env::current_exe()+ .ok()+ .and_then(|p| p.to_str().map(str::to_owned))+ .unwrap_or_else(|| "cua-driver".to_owned());++ match client {+ None | Some("") => {+ println!(r#"{{+ "mcpServers": {{+ "cua-driver": {{+ "command": "{binary}",+ "args": ["mcp"]+ }}+ }}+}}"#);+ }+ Some("claude") => {+ println!("claude mcp add --transport stdio cua-driver -- {binary} mcp");+ }+ Some("codex") => {+ println!("codex mcp add cua-driver -- {binary} mcp");+ }+ Some("cursor") => {+ println!(r#"{{+ "mcpServers": {{+ "cua-driver": {{+ "command": "{binary}",+ "args": ["mcp"],+ "type": "stdio"+ }}+ }}+}}"#);+ }+ Some("openclaw") => {+ println!("openclaw mcp set cua-driver '{{\"command\":\"{binary}\",\"args\":[\"mcp\"]}}'");+ }+ Some("opencode") => {+ println!(r#"// paste under "mcp" in opencode.json:+{{+ "$schema": "https://opencode.ai/config.json",+ "mcp": {{+ "cua-driver": {{+ "type": "local",+ "command": ["{binary}", "mcp"],+ "enabled": true+ }}+ }}+}}"#);+ }+ Some("hermes") => {+ println!("# paste under mcp_servers in ~/.hermes/config.yaml,");+ println!("# then run /reload-mcp inside Hermes:");+ println!("mcp_servers:");+ println!(" cua-driver:");+ println!(" command: \"{binary}\"");+ println!(" args: [\"mcp\"]");+ }+ Some("pi") => {+ println!(+ "Pi (badlogic/pi-mono) does not support MCP natively — the author\n\+ has stated MCP support will not be added for context-budget reasons.\n\n\+ Use cua-driver as a plain CLI from inside Pi instead:\n\n\+ {binary} list_apps\n\+ {binary} click '{{\"pid\": 1234, \"x\": 100, \"y\": 200}}'\n\+ {binary} --help # full tool catalog\n\n\+ Each call is one-shot and returns JSON / text on stdout, which is\n\+ exactly the shape Pi is designed around."+ );+ }+ Some(other) => {+ eprintln!("Unknown client '{other}'. Valid: claude, codex, cursor, openclaw, opencode, hermes, pi.");+ process::exit(2);+ }+ }+}++/// Invoke a tool, forwarding to a running daemon if one is reachable;+/// otherwise runs in-process. Prints result to stdout on success, error+/// to stderr on failure. Exits 1 if the tool returned an error result.+/// When `screenshot_out_file` is provided, image content is written there+/// instead of emitted as base64 on stdout.+///+/// `socket` — override the daemon socket path (from --socket flag).+pub fn run_call(+ registry: std::sync::Arc<ToolRegistry>,+ tool: &str,+ json_args: Option<serde_json::Value>,+ screenshot_out_file: Option<String>,+) {+ // Daemon forwarding: if a daemon is listening, proxy the request+ // through it so AppStateEngine's element_index cache is shared.+ let socket_path = crate::serve::default_socket_path();+ if crate::serve::is_daemon_listening(&socket_path) {+ let args_for_daemon = json_args.clone()+ .unwrap_or(serde_json::Value::Object(serde_json::Map::new()));+ let req = crate::serve::DaemonRequest {+ method: "call".into(),+ name: Some(tool.to_owned()),+ args: Some(args_for_daemon),+ };+ match crate::serve::send_request(&socket_path, &req) {+ Ok(resp) => {+ if resp.ok {+ if let Some(result) = resp.result {+ // Handle image write if --screenshot-out-file was given.+ let mut printed = false;+ if let Some(content) = result.get("content").and_then(|v| v.as_array()) {+ for item in content {+ if item.get("type").and_then(|v| v.as_str()) == Some("image") {+ if let Some(ref path) = screenshot_out_file {+ if let Some(b64) = item.get("data").and_then(|v| v.as_str()) {+ use base64::Engine as _;+ match base64::engine::general_purpose::STANDARD.decode(b64) {+ Ok(bytes) => { let _ = std::fs::write(path, &bytes); }+ Err(e) => eprintln!("--screenshot-out-file: {e}"),+ }+ }+ }+ }+ }+ }+ if let Some(sc) = result.get("structuredContent") {+ let pretty = serde_json::to_string_pretty(sc)+ .unwrap_or_else(|_| sc.to_string());+ println!("{pretty}");+ printed = true;+ }+ if !printed {+ if let Some(content) = result.get("content").and_then(|v| v.as_array()) {+ for item in content {+ if item.get("type").and_then(|v| v.as_str()) == Some("text") {+ if let Some(text) = item.get("text").and_then(|v| v.as_str()) {+ println!("{text}");+ }+ }+ }+ }+ }+ }+ return;+ } else {+ if let Some(err) = resp.error {+ eprintln!("{err}");+ }+ process::exit(resp.exit_code.unwrap_or(1));+ }+ }+ Err(e) => {+ // Daemon became unreachable mid-call — fall through to in-process.+ tracing::debug!("Daemon forwarding failed ({e}), running in-process");+ }+ }+ }+ if registry.get_def(tool).is_none() {+ eprintln!("Unknown tool: {tool}");+ eprintln!("Run `cua-driver list-tools` to see available tools.");+ process::exit(64);+ }++ let rt = tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()+ .expect("tokio runtime");++ let args = json_args.unwrap_or(serde_json::Value::Object(serde_json::Map::new()));+ let tool_name = tool.to_string();+ let out_path = screenshot_out_file;+ let is_error = rt.block_on(async move {+ let result = registry.invoke(&tool_name, args).await;+ let is_err = result.is_error.unwrap_or(false);++ // Emit content.+ let mut has_printed = false;+ let mut image_b64: Option<(String, String)> = None; // (base64, mime)+ for item in &result.content {+ match item {+ Content::Text { text, .. } => {+ if is_err {+ eprintln!("{text}");+ } else {+ // Only print text when there is no structuredContent+ // (structuredContent path prints below).+ if result.structured_content.is_none() {+ println!("{text}");+ has_printed = true;+ }+ }+ }+ Content::Image { data, mime_type, .. } => {+ image_b64 = Some((data.clone(), mime_type.clone()));+ }+ }+ }++ // If --screenshot-out-file was provided, write the image there+ // and suppress it from the JSON output (same as Swift reference).+ if let Some(ref path) = out_path {+ if let Some((b64, _mime)) = image_b64.take() {+ use base64::Engine as _;+ match base64::engine::general_purpose::STANDARD.decode(&b64) {+ Ok(bytes) => {+ if let Err(e) = std::fs::write(path, &bytes) {+ eprintln!("--screenshot-out-file: failed to write {path}: {e}");+ }+ }+ Err(e) => {+ eprintln!("--screenshot-out-file: base64 decode failed: {e}");+ }+ }+ } else {+ eprintln!("--screenshot-out-file: no image content in tool response; file not written");+ }+ }++ // If there's structuredContent, print it as JSON (with image merged in if no out_path).+ if let Some(sc) = &result.structured_content {+ if !is_err {+ let mut obj = sc.clone();+ if out_path.is_none() {+ if let Some((b64, mime)) = image_b64 {+ if let serde_json::Value::Object(ref mut map) = obj {+ map.insert("screenshot_png_b64".into(), serde_json::Value::String(b64));+ map.insert("screenshot_mime_type".into(), serde_json::Value::String(mime));+ }+ }+ }+ let pretty = serde_json::to_string_pretty(&obj)+ .unwrap_or_else(|_| obj.to_string());+ println!("{pretty}");+ has_printed = true;+ }+ }++ let _ = has_printed;+ is_err+ });++ if is_error {+ process::exit(1);+ }+}++/// `cua-driver recording <start|stop|status>` — wrapper around+/// `set_recording` / `get_recording_state` tools on the running daemon.+///+/// Requires a running daemon (`cua-driver serve`) because recording+/// state lives in-process.+pub fn run_recording_cmd(subcommand: &str, args: &[String], socket: Option<&str>) {+ let socket_path = socket+ .map(str::to_owned)+ .unwrap_or_else(crate::serve::default_socket_path);++ if !crate::serve::is_daemon_listening(&socket_path) {+ eprintln!(+ "cua-driver daemon is not running.\n\+ Start it first with: cua-driver serve"+ );+ process::exit(1);+ }++ match subcommand {+ "start" => {+ let output_dir = match args.first() {+ Some(d) => {+ // Expand ~ manually.+ if d.starts_with('~') {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ d.replacen('~', &home, 1)+ } else {+ d.clone()+ }+ }+ None => {+ eprintln!("Usage: cua-driver recording start <output-dir>");+ process::exit(64);+ }+ };++ // Create the directory if it doesn't exist.+ if let Err(e) = std::fs::create_dir_all(&output_dir) {+ eprintln!("Failed to create output directory {output_dir}: {e}");+ process::exit(1);+ }++ let req = crate::serve::DaemonRequest {+ method: "call".into(),+ name: Some("set_recording".into()),+ args: Some(serde_json::json!({+ "enabled": true,+ "output_dir": output_dir+ })),+ };+ match crate::serve::send_request(&socket_path, &req) {+ Ok(resp) if resp.ok => {+ println!("Recording started → {output_dir}");+ // Query state to show next_turn.+ let state_req = crate::serve::DaemonRequest {+ method: "call".into(),+ name: Some("get_recording_state".into()),+ args: Some(serde_json::json!({})),+ };+ if let Ok(sr) = crate::serve::send_request(&socket_path, &state_req) {+ if let Some(result) = sr.result {+ let sc = result.get("structuredContent")+ .or_else(|| result.get("structured_content"));+ if let Some(next_turn) = sc.and_then(|s| s.get("next_turn")).and_then(|v| v.as_u64()) {+ println!("Next turn: {next_turn:05}");+ }+ }+ }+ }+ Ok(resp) => {+ if let Some(e) = resp.error { eprintln!("{e}"); }+ process::exit(1);+ }+ Err(e) => { eprintln!("recording start: {e}"); process::exit(1); }+ }+ }++ "stop" => {+ let req = crate::serve::DaemonRequest {+ method: "call".into(),+ name: Some("set_recording".into()),+ args: Some(serde_json::json!({ "enabled": false })),+ };+ match crate::serve::send_request(&socket_path, &req) {+ Ok(resp) if resp.ok => println!("Recording stopped."),+ Ok(resp) => {+ if let Some(e) = resp.error { eprintln!("{e}"); }+ process::exit(1);+ }+ Err(e) => { eprintln!("recording stop: {e}"); process::exit(1); }+ }+ }++ "status" | "" => {+ let req = crate::serve::DaemonRequest {+ method: "call".into(),+ name: Some("get_recording_state".into()),+ args: Some(serde_json::json!({})),+ };+ match crate::serve::send_request(&socket_path, &req) {+ Ok(resp) if resp.ok => {+ if let Some(result) = resp.result {+ let sc = result.get("structuredContent")+ .or_else(|| result.get("structured_content"))+ .cloned()+ .unwrap_or(serde_json::json!({}));+ let enabled = sc.get("enabled").and_then(|v| v.as_bool()).unwrap_or(false);+ let out_dir = sc.get("output_dir").and_then(|v| v.as_str()).unwrap_or("(none)");+ let next_turn = sc.get("next_turn").and_then(|v| v.as_u64()).unwrap_or(0);+ println!("Recording: {}", if enabled { "enabled" } else { "disabled" });+ if enabled {+ println!(" output_dir: {out_dir}");+ println!(" next_turn: {next_turn:05}");+ }+ }+ }+ Ok(resp) => {+ if let Some(e) = resp.error { eprintln!("{e}"); }+ process::exit(1);+ }+ Err(e) => { eprintln!("recording status: {e}"); process::exit(1); }+ }+ }++ other => {+ eprintln!("Unknown recording subcommand '{other}'. Valid: start <dir>, stop, status");+ process::exit(64);+ }+ }+}++/// `cua-driver update [--apply]` — check for a newer release and optionally apply it.+///+/// Uses `curl` to query the GitHub releases API (same as Swift reference).+/// Pass `--apply` to download and install via the canonical install.sh.+pub fn run_update_cmd(apply: bool) {+ let current = env!("CARGO_PKG_VERSION");+ println!("Current version: {current}");+ println!("Checking for updates…");++ let latest = fetch_latest_version();+ match latest {+ None => {+ println!("Could not reach GitHub — check your connection and try again.");+ process::exit(1);+ }+ Some(v) if !is_version_newer(&v, current) => {+ println!("Already up to date.");+ }+ Some(v) => {+ println!("New version available: {v}");++ if !apply {+ println!();+ println!("Run with --apply to download and install it:");+ println!(" cua-driver update --apply");+ println!();+ println!("Or reinstall directly:");+ println!(" curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver-rs/scripts/install.sh | bash");+ return;+ }++ println!("Downloading and installing cua-driver {v}…");+ let status = std::process::Command::new("bash")+ .arg("-c")+ .arg("curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver-rs/scripts/install.sh | bash")+ .status();+ match status {+ Ok(s) if s.success() => {}+ Ok(s) => {+ println!("Installation failed (exit {}). Run the command above manually.", s.code().unwrap_or(1));+ process::exit(s.code().unwrap_or(1));+ }+ Err(e) => {+ eprintln!("Failed to run installer: {e}");+ process::exit(1);+ }+ }+ }+ }+}++/// Fetch the latest `cua-driver-v*` release tag from GitHub using curl.+/// Returns the version string (e.g. "0.1.2") or `None` on any error.+fn fetch_latest_version() -> Option<String> {+ let out = std::process::Command::new("curl")+ .args(["-s", "--max-time", "4",+ "-H", "Accept: application/vnd.github+json",+ "https://api.github.com/repos/trycua/cua/releases?per_page=40"])+ .output()+ .ok()?;+ if !out.status.success() { return None; }+ let text = String::from_utf8_lossy(&out.stdout);+ let releases: serde_json::Value = serde_json::from_str(&text).ok()?;+ // `cua-driver-rs-v*` releases are distinct from the Swift `cua-driver-v*`+ // releases on the same repo — must filter by the Rust-port prefix or+ // we'd recommend the Swift binary instead.+ let prefix = "cua-driver-rs-v";+ let mut versions: Vec<String> = releases.as_array()?.iter()+ .filter_map(|r| {+ let tag = r["tag_name"].as_str()?;+ if !tag.starts_with(prefix) { return None; }+ if r["draft"].as_bool().unwrap_or(false) { return None; }+ if r["prerelease"].as_bool().unwrap_or(false) { return None; }+ Some(tag[prefix.len()..].to_owned())+ })+ .collect();+ // Sort descending (newest first).+ versions.sort_by(|a, b| compare_versions(b, a));+ versions.into_iter().next()+}++/// True when `candidate` is strictly newer than `current` (semver compare).+fn is_version_newer(candidate: &str, current: &str) -> bool {+ compare_versions(candidate, current) == std::cmp::Ordering::Greater+}++fn compare_versions(a: &str, b: &str) -> std::cmp::Ordering {+ let parts = |s: &str| -> Vec<u64> {+ s.split('.').filter_map(|p| p.parse().ok()).collect()+ };+ let pa = parts(a);+ let pb = parts(b);+ for (x, y) in pa.iter().zip(pb.iter()) {+ match x.cmp(y) {+ std::cmp::Ordering::Equal => continue,+ other => return other,+ }+ }+ pa.len().cmp(&pb.len())+}++/// `cua-driver dump-docs [--pretty]` — output all MCP tool schemas as JSON.+pub fn run_dump_docs(registry: &ToolRegistry, pretty: bool) {+ run_dump_docs_with_type(registry, pretty, "all")+}++/// Output documentation as JSON. `doc_type` is one of:+/// - `"mcp"` — only MCP tool docs (`{version, tools: [...]}`)+/// - `"cli"` — CLI docs (stub on Rust — Swift extracts via swift-argument-parser+/// introspection which has no clap analogue without a bigger refactor)+/// - `"all"` — `{cli, mcp}` matching Swift `CombinedDocs`+pub fn run_dump_docs_with_type(registry: &ToolRegistry, pretty: bool, doc_type: &str) {+ // Each MCP tool: `{name, description, input_schema}` (Swift's MCPToolDoc+ // shape — Rust adds read_only/destructive/idempotent as intentional+ // extras documented in PARITY.md).+ let tools: Vec<serde_json::Value> = registry.iter_defs()+ .map(|(_, def)| serde_json::json!({+ "name": def.name,+ "description": def.description,+ "input_schema": def.input_schema,+ "read_only": def.read_only,+ "destructive": def.destructive,+ "idempotent": def.idempotent,+ }))+ .collect();+ let mcp = serde_json::json!({+ "version": env!("CARGO_PKG_VERSION"),+ "tools": tools,+ });++ // CLI docs are intentionally a thin stub on Rust — full extraction+ // would require introspecting clap's command graph which we don't use+ // (cli.rs uses hand-rolled arg matching). Documented in PARITY.md.+ let cli_stub = serde_json::json!({+ "version": env!("CARGO_PKG_VERSION"),+ "commands": [],+ "_note": "CLI introspection not implemented on Rust port. Use `cua-driver --help` for CLI docs.",+ });++ let doc = match doc_type {+ "cli" => cli_stub,+ "mcp" => mcp,+ _ => serde_json::json!({ "cli": cli_stub, "mcp": mcp }),+ };+ let out = if pretty {+ serde_json::to_string_pretty(&doc)+ } else {+ serde_json::to_string(&doc)+ };+ println!("{}", out.unwrap_or_else(|_| "{}".into()));+}++/// `cua-driver diagnose` — print a paste-able bundle-path / install-layout / TCC report.+///+/// Mirrors Swift `DiagnoseCommand`. Covers:+/// - running process identity (path, pid, version)+/// - codesign info (cdhash, team-id, authority) via `codesign -dvvv`+/// - AX + screen recording TCC status (check_permissions tool)+/// - install layout (/Applications/CuaDriver.app, ~/.local/bin/cua-driver)+/// - TCC DB rows for com.trycua.driver (sqlite3, best-effort)+/// - config + state paths with existence booleans+pub fn run_diagnose_cmd(registry: std::sync::Arc<ToolRegistry>) {+ let sections = [+ diagnose_runtime_section(),+ diagnose_signature_section(),+ diagnose_tcc_section(registry),+ diagnose_install_layout_section(),+ diagnose_tcc_db_section(),+ diagnose_config_paths_section(),+ ];+ println!("{}", sections.join("\n\n"));+}++fn diagnose_runtime_section() -> String {+ let exe = std::env::current_exe()+ .ok()+ .and_then(|p| p.to_str().map(str::to_owned))+ .unwrap_or_else(|| "<unknown>".into());+ let argv0 = std::env::args().next().unwrap_or_else(|| "<unknown>".into());+ let pid = std::process::id();+ let version = env!("CARGO_PKG_VERSION");+ format!(+ "## running process\n\+ version: {version}\n\+ argv[0]: {argv0}\n\+ executablePath: {exe}\n\+ pid: {pid}"+ )+}++fn diagnose_signature_section() -> String {+ let exe = std::env::current_exe()+ .ok()+ .and_then(|p| p.to_str().map(str::to_owned))+ .unwrap_or_default();++ let out = std::process::Command::new("codesign")+ .args(["-dvvv", &exe])+ .output();++ let mut cdhash = "<unknown>".to_owned();+ let mut team_id = "<none — ad-hoc signed?>".to_owned();+ let mut authority = "<none — ad-hoc signed?>".to_owned();++ if let Ok(out) = out {+ // codesign prints to stderr+ let text = String::from_utf8_lossy(&out.stderr);+ for line in text.lines() {+ if let Some(rest) = line.strip_prefix("CDHash=") {+ cdhash = rest.trim().to_owned();+ } else if let Some(rest) = line.strip_prefix("TeamIdentifier=") {+ team_id = rest.trim().to_owned();+ } else if let Some(rest) = line.strip_prefix("Authority=") {+ // Only take the first (leaf certificate) authority line.+ if authority == "<none — ad-hoc signed?>" {+ authority = rest.trim().to_owned();+ }+ }+ }+ }++ format!(+ "## running process signature\n\+ cdhash: {cdhash}\n\+ teamID: {team_id}\n\+ authority: {authority}"+ )+}++fn diagnose_tcc_section(registry: std::sync::Arc<ToolRegistry>) -> String {+ // Call check_permissions in-process (quick, read-only).+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build();++ let (ax, sr) = if let Ok(rt) = rt {+ rt.block_on(async {+ let result = registry.invoke("check_permissions",+ serde_json::Value::Object(Default::default())).await;+ if let Some(sc) = &result.structured_content {+ let ax = sc.get("accessibility")+ .and_then(|v| v.as_bool())+ .unwrap_or(false);+ let sr = sc.get("screen_recording")+ .and_then(|v| v.as_bool())+ .unwrap_or(false);+ (ax, sr)+ } else {+ (false, false)+ }+ })+ } else {+ (false, false)+ };++ format!(+ "## tcc probes (live process)\n\+ accessibility (AXIsProcessTrusted): {ax}\n\+ screen recording (SCShareableContent): {sr}\n\n\+ if the UI disagrees with these booleans the live process is fine —\n\+ the issue is elsewhere (wrong bundle granted, stale cdhash, etc)."+ )+}++fn diagnose_install_layout_section() -> String {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ let mut lines = vec!["## install layout".to_owned()];++ let app_path = "/Applications/CuaDriver.app";+ let app_exists = std::path::Path::new(app_path).exists();+ lines.push(format!("bundle: {app_path} exists={app_exists}"));+ if app_exists {+ // Codesign info for the bundle.+ if let Ok(out) = std::process::Command::new("codesign")+ .args(["-dvvv", app_path])+ .output()+ {+ let text = String::from_utf8_lossy(&out.stderr);+ let mut cdhash = "<unknown>";+ let mut team_id = "<none>";+ let mut authority = "<none>";+ for line in text.lines() {+ if let Some(r) = line.strip_prefix("CDHash=") {+ cdhash = r.trim();+ } else if let Some(r) = line.strip_prefix("TeamIdentifier=") {+ team_id = r.trim();+ } else if line.starts_with("Authority=") && authority == "<none>" {+ authority = line.trim_start_matches("Authority=").trim();+ }+ }+ lines.push(format!(" cdhash: {cdhash}"));+ lines.push(format!(" teamID: {team_id}"));+ lines.push(format!(" authority: {authority}"));+ }+ }++ let cli_paths = [+ ("symlink", format!("{home}/.local/bin/cua-driver")),+ ("legacy symlink", "/usr/local/bin/cua-driver".to_owned()),+ ];+ for (label, path) in &cli_paths {+ let exists = std::path::Path::new(path).exists();+ lines.push(format!("{label}: {path} exists={exists}"));+ if exists {+ if let Ok(target) = std::fs::read_link(path) {+ lines.push(format!(" resolves to: {}", target.display()));+ }+ }+ }++ let stale = format!("{home}/Applications/CuaDriver.app");+ if std::path::Path::new(&stale).exists() {+ lines.push(format!(+ "stale: {stale} \u{2190} old install-local.sh path, consider removing"+ ));+ }++ lines.join("\n")+}++fn diagnose_tcc_db_section() -> String {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ let db = format!(+ "{home}/Library/Application Support/com.apple.TCC/TCC.db"+ );+ let sql = "SELECT service, client, client_type, auth_value, auth_reason, \+ hex(csreq) AS csreq_hex FROM access WHERE client='com.trycua.driver';";++ let mut lines = vec!["## tcc database rows for com.trycua.driver".to_owned()];+ lines.push(format!("(reading {db} — best-effort; system TCC DB requires FDA)"));+ lines.push(String::new());++ match std::process::Command::new("sqlite3")+ .args(["-header", "-column", &db, sql])+ .output()+ {+ Ok(out) => {+ let stdout = String::from_utf8_lossy(&out.stdout);+ let stderr = String::from_utf8_lossy(&out.stderr);+ let trimmed = stdout.trim();+ if !trimmed.is_empty() {+ lines.push(trimmed.to_owned());+ } else if out.status.code() != Some(0) && !stderr.trim().is_empty() {+ lines.push(format!("(sqlite3 failed: {})", stderr.trim()));+ } else {+ lines.push("(no rows — either grants never made it to TCC, or they live".into());+ lines.push(" in the system DB which requires Full Disk Access to read.)".into());+ }+ }+ Err(e) => {+ lines.push(format!("(could not launch sqlite3: {e})"));+ }+ }++ lines.push(String::new());+ lines.push("# auth_value legend: 0=denied 2=allowed".into());+ lines.push("# services: kTCCServiceAccessibility, kTCCServiceScreenCapture".into());+ lines.join("\n")+}++fn diagnose_config_paths_section() -> String {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ let paths: &[(&str, String)] = &[+ ("user data dir", format!("{home}/.cua-driver")),+ ("config cache", format!("{home}/Library/Caches/cua-driver")),+ ("telemetry id", format!("{home}/.cua-driver/.telemetry_id")),+ ("updater plist", format!("{home}/Library/LaunchAgents/com.trycua.cua_driver_updater.plist")),+ ("daemon plist", format!("{home}/Library/LaunchAgents/com.trycua.cua_driver_daemon.plist")),+ ];+ let mut lines = vec!["## config + state paths".to_owned()];+ for (label, path) in paths {+ let exists = std::path::Path::new(path).exists();+ lines.push(format!("{:<18} exists={exists} {path}", label));+ }+ lines.join("\n")+}++/// Path to the persistent JSON config file (`~/.cua-driver/config.json`).+fn config_file_path() -> std::path::PathBuf {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ std::path::PathBuf::from(format!("{home}/.cua-driver/config.json"))+}++/// Read persisted config from disk. Returns an empty object if absent/unreadable.+fn read_config_file() -> serde_json::Value {+ let path = config_file_path();+ std::fs::read_to_string(&path)+ .ok()+ .and_then(|s| serde_json::from_str(&s).ok())+ .unwrap_or_else(|| serde_json::json!({}))+}++/// Write a single key/value into the persisted config file.+fn write_config_file(key: &str, value: &serde_json::Value) {+ let path = config_file_path();+ if let Some(parent) = path.parent() {+ let _ = std::fs::create_dir_all(parent);+ }+ let mut cfg = read_config_file();+ if let serde_json::Value::Object(ref mut map) = cfg {+ map.insert(key.to_owned(), value.clone());+ }+ if let Ok(json) = serde_json::to_string_pretty(&cfg) {+ let _ = std::fs::write(&path, json);+ }+}++/// `cua-driver config [show|get|set|reset] [key] [value]`+///+/// Thin wrapper around the `get_config` / `set_config` MCP tools.+/// Forwards to the daemon when one is reachable, otherwise runs in-process.+pub fn run_config_cmd(+ registry: std::sync::Arc<ToolRegistry>,+ subcommand: Option<&str>,+ key: Option<&str>,+ value: Option<&str>,+ socket: Option<&str>,+) {+ let socket_path = socket+ .map(str::to_owned)+ .unwrap_or_else(crate::serve::default_socket_path);++ let rt = tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()+ .expect("tokio runtime");++ match subcommand.unwrap_or("show") {+ "show" | "" => {+ // Print full config as pretty JSON.+ let config = rt.block_on(async {+ registry.invoke("get_config", serde_json::json!({})).await+ });+ if let Some(sc) = config.structured_content {+ println!("{}", serde_json::to_string_pretty(&sc).unwrap_or_else(|_| sc.to_string()));+ } else {+ eprintln!("get_config: no structured content returned");+ process::exit(1);+ }+ }++ "get" => {+ let key = match key {+ Some(k) => k,+ None => {+ eprintln!("Usage: cua-driver config get <key>");+ eprintln!("Keys: capture_mode, max_image_dimension, version, platform");+ process::exit(64);+ }+ };+ // Try daemon first.+ if crate::serve::is_daemon_listening(&socket_path) {+ let req = crate::serve::DaemonRequest {+ method: "call".into(),+ name: Some("get_config".into()),+ args: Some(serde_json::json!({})),+ };+ if let Ok(resp) = crate::serve::send_request(&socket_path, &req) {+ if resp.ok {+ if let Some(result) = resp.result {+ if let Some(sc) = result.get("structuredContent") {+ if let Some(v) = sc.get(key) {+ println!("{v}");+ return;+ }+ }+ }+ }+ }+ }+ // In-process: merge persisted file config over in-memory defaults.+ let config = rt.block_on(async {+ registry.invoke("get_config", serde_json::json!({})).await+ });+ let mut sc = match config.structured_content {+ Some(v) => v,+ None => {+ eprintln!("get_config: no structured content returned");+ process::exit(1);+ }+ };+ // Overlay persisted values from the config file.+ let file_cfg = read_config_file();+ if let (serde_json::Value::Object(sc_map), serde_json::Value::Object(file_map)) =+ (&mut sc, file_cfg)+ {+ for (k, v) in file_map {+ sc_map.insert(k, v);+ }+ }+ // Support dotted key paths like "agent_cursor.enabled".+ let v = if key.contains('.') {+ let mut parts = key.splitn(2, '.');+ let parent = parts.next().unwrap();+ let child = parts.next().unwrap();+ sc.get(parent).and_then(|obj| obj.get(child)).cloned()+ } else {+ sc.get(key).cloned()+ };+ if let Some(v) = v {+ println!("{}", match &v { serde_json::Value::String(s) => s.clone(), other => other.to_string() });+ } else {+ eprintln!("Unknown config key: {key}");+ eprintln!("Available keys: capture_mode, max_image_dimension, version, platform, agent_cursor.enabled");+ process::exit(64);+ }+ }++ "set" => {+ let key = match key {+ Some(k) => k,+ None => {+ eprintln!("Usage: cua-driver config set <key> <value>");+ process::exit(64);+ }+ };+ let value = match value {+ Some(v) => v,+ None => {+ eprintln!("Usage: cua-driver config set {key} <value>");+ process::exit(64);+ }+ };+ // Parse value: try JSON, fall back to string.+ let parsed_value: serde_json::Value =+ serde_json::from_str(value).unwrap_or_else(|_| serde_json::Value::String(value.to_owned()));+ let args = serde_json::json!({ key: parsed_value });++ // Try daemon first.+ if crate::serve::is_daemon_listening(&socket_path) {+ let req = crate::serve::DaemonRequest {+ method: "call".into(),+ name: Some("set_config".into()),+ args: Some(args.clone()),+ };+ if let Ok(resp) = crate::serve::send_request(&socket_path, &req) {+ if resp.ok {+ println!("Config updated.");+ // Show current state.+ let req2 = crate::serve::DaemonRequest {+ method: "call".into(),+ name: Some("get_config".into()),+ args: Some(serde_json::json!({})),+ };+ if let Ok(r2) = crate::serve::send_request(&socket_path, &req2) {+ if let Some(result) = r2.result {+ if let Some(sc) = result.get("structuredContent") {+ println!("{}", serde_json::to_string_pretty(sc).unwrap_or_default());+ }+ }+ }+ return;+ } else if let Some(e) = resp.error {+ eprintln!("{e}");+ process::exit(1);+ }+ }+ }+ // In-process.+ let result = rt.block_on(async {+ registry.invoke("set_config", args).await+ });+ if result.is_error.unwrap_or(false) {+ for item in &result.content {+ if let mcp_server::protocol::Content::Text { text, .. } = item {+ eprintln!("{text}");+ }+ }+ process::exit(1);+ }+ // Persist the value to disk so future CLI invocations can read it.+ write_config_file(key, &parsed_value);+ // Print updated config.+ let config = rt.block_on(async {+ registry.invoke("get_config", serde_json::json!({})).await+ });+ if let Some(sc) = config.structured_content {+ println!("{}", serde_json::to_string_pretty(&sc).unwrap_or_else(|_| sc.to_string()));+ }+ }++ "reset" => {+ // Reset to defaults by calling set_config with empty args.+ // The set_config tool keeps existing values if keys are absent,+ // so we send the known defaults explicitly.+ let defaults = serde_json::json!({+ "capture_mode": "som",+ "max_image_dimension": 0+ });+ let result = rt.block_on(async {+ registry.invoke("set_config", defaults).await+ });+ if result.is_error.unwrap_or(false) {+ eprintln!("config reset failed");+ process::exit(1);+ }+ println!("Config reset to defaults.");+ let config = rt.block_on(async {+ registry.invoke("get_config", serde_json::json!({})).await+ });+ if let Some(sc) = config.structured_content {+ println!("{}", serde_json::to_string_pretty(&sc).unwrap_or_else(|_| sc.to_string()));+ }+ }++ other => {+ eprintln!("Unknown config subcommand '{other}'. Valid: show, get <key>, set <key> <value>, reset");+ process::exit(64);+ }+ }+}++/// `cua-driver doctor` — clean up stale install bits left from older cua-driver versions.+///+/// Mirrors Swift `DoctorCommand`:+/// - Removes the legacy LaunchAgent plist (~/Library/LaunchAgents/com.trycua.cua_driver_updater.plist)+/// via `launchctl unload` then `fs::remove_file`.+/// - Removes the legacy update script (/usr/local/bin/cua-driver-update) if writable;+/// otherwise prints the `sudo rm` command the user must run manually.+pub fn run_doctor_cmd() {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ let legacy_plist = format!("{home}/Library/LaunchAgents/com.trycua.cua_driver_updater.plist");+ let legacy_script = "/usr/local/bin/cua-driver-update";++ let mut cleaned_anything = false;++ // — Legacy LaunchAgent plist —+ if std::path::Path::new(&legacy_plist).exists() {+ cleaned_anything = true;+ println!("Removing legacy LaunchAgent: {legacy_plist}");++ // Try to unload it first (ignore errors — may not be loaded).+ let _ = std::process::Command::new("launchctl")+ .args(["unload", &legacy_plist])+ .status();++ match std::fs::remove_file(&legacy_plist) {+ Ok(()) => println!(" ✓ Removed."),+ Err(e) => eprintln!(" ✗ Failed to remove {legacy_plist}: {e}"),+ }+ }++ // — Legacy update script —+ if std::path::Path::new(legacy_script).exists() {+ cleaned_anything = true;+ println!("Removing legacy update script: {legacy_script}");+ match std::fs::remove_file(legacy_script) {+ Ok(()) => println!(" ✓ Removed."),+ Err(_) => {+ // Root-owned — user must sudo.+ println!(" Could not remove (root-owned). Run manually:");+ println!(" sudo rm -f {legacy_script}");+ }+ }+ }++ if !cleaned_anything {+ println!("Nothing to clean — install is up to date.");+ }+}++// ── helpers ──────────────────────────────────────────────────────────────────++/// Read JSON from stdin when stdin is a pipe (non-interactive). Returns `None`+/// when stdin is a terminal or the input isn't valid JSON.+/// Matches Swift's "If omitted, reads from stdin when stdin is a pipe."+fn read_stdin_json() -> Option<serde_json::Value> {+ use std::io::{self, IsTerminal, Read};+ let stdin = io::stdin();+ if stdin.is_terminal() {+ return None;+ }+ let mut buf = String::new();+ stdin.lock().read_to_string(&mut buf).ok()?;+ serde_json::from_str(buf.trim()).ok()+}++fn first_sentence(text: &str) -> String {+ let trimmed = text.trim();+ if trimmed.is_empty() { return String::new(); }+ let flat: String = trimmed+ .splitn(3, "\n\n")+ .next()+ .unwrap_or(trimmed)+ .split('\n')+ .collect::<Vec<_>>()+ .join(" ");+ let mut sentence = String::new();+ let mut prev = ' ';+ for ch in flat.chars() {+ if (prev == '.' || prev == '?' || prev == '!') && ch == ' ' {+ break;+ }+ sentence.push(ch);+ prev = ch;+ }+ let mut s = sentence.trim().to_string();+ if s.ends_with('.') { s.pop(); }+ s+}diff --git a/libs/cua-driver-rs/crates/cua-driver/src/main.rs b/libs/cua-driver-rs/crates/cua-driver/src/main.rsnew file mode 100644index 0000000000..c8f8504187--- /dev/null+++ b/libs/cua-driver-rs/crates/cua-driver/src/main.rs@@ -0,0 +1,467 @@+//! cua-driver-rs — cross-platform background computer-use automation daemon.+//!+//! Runs as a MCP JSON-RPC 2.0 server over stdio. The platform backend is+//! selected at compile time via conditional compilation.+//!+//! Extra CLI flags (consumed here, not by MCP):+//! --cursor-icon <path.svg|.ico|.png> custom cursor shape+//! --cursor-id <id> multi-cursor instance id+//! --cursor-palette <name> named colour palette+//! --no-overlay start with overlay disabled+//! --glide-ms <f64> glide duration override+//! --dwell-ms <f64> post-click dwell override+//! --idle-hide-ms <f64> idle-hide timeout override+//!+//! ## macOS threading model+//!+//! AppKit requires the main thread. On macOS the entry-point is a plain+//! `fn main()` that:+//! 1. Initialises the cursor overlay channel synchronously (so+//! `run_on_main_thread` always finds it ready).+//! 2. Spawns a background tokio thread for the MCP server.+//! 3. Calls `platform_macos::cursor::overlay::run_on_main_thread()` which+//! starts `NSApplication.run()` and the 60 fps render loop.+//!+//! On all other platforms `#[tokio::main]` is used directly.++mod cli;+mod serve;++use std::sync::Arc;++fn init_logging() {+ use tracing_subscriber::EnvFilter;+ tracing_subscriber::fmt()+ .with_writer(std::io::stderr)+ .with_env_filter(+ EnvFilter::from_env("CUA_LOG")+ .add_directive(tracing::Level::WARN.into()),+ )+ .init();+}++// ── macOS entry-point ─────────────────────────────────────────────────────++#[cfg(target_os = "macos")]+fn main() {+ init_logging();++ // ── CLI subcommand dispatch ──────────────────────────────────────────────+ // Handled before AppKit init so `list-tools` / `describe` / `call` exit+ // cleanly without starting the overlay or NSApplication.+ match cli::parse_command() {+ cli::Command::ListTools => {+ let reg = Arc::new(platform_macos::register_tools());+ cli::run_list_tools(®);+ return;+ }+ cli::Command::Describe(name) => {+ let reg = Arc::new(platform_macos::register_tools());+ cli::run_describe(®, &name);+ return;+ }+ cli::Command::McpConfig { client } => {+ cli::run_mcp_config(client.as_deref());+ return;+ }+ cli::Command::Call { tool, json_args, screenshot_out_file } => {+ // Register callbacks (needed if the tool does screenshots/recording).+ mcp_server::recording::set_screenshot_fn(|window_id, pid| {+ if let Some(wid) = window_id {+ platform_macos::capture::screenshot_window_bytes(wid as u32).ok()+ } else if let Some(p) = pid {+ platform_macos::windows::resolve_main_window_id(p as i32).ok()+ .and_then(|wid| platform_macos::capture::screenshot_window_bytes(wid).ok())+ } else {+ platform_macos::capture::screenshot_display_bytes().ok()+ }+ });+ mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| {+ platform_macos::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()+ });+ let reg = Arc::new(platform_macos::register_tools());+ reg.init_self_weak();+ cli::run_call(reg, &tool, json_args, screenshot_out_file);+ return;+ }+ cli::Command::Serve { socket } => {+ mcp_server::recording::set_screenshot_fn(|window_id, pid| {+ if let Some(wid) = window_id {+ platform_macos::capture::screenshot_window_bytes(wid as u32).ok()+ } else if let Some(p) = pid {+ platform_macos::windows::resolve_main_window_id(p as i32).ok()+ .and_then(|wid| platform_macos::capture::screenshot_window_bytes(wid).ok())+ } else {+ platform_macos::capture::screenshot_display_bytes().ok()+ }+ });+ mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| {+ platform_macos::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()+ });+ let reg = Arc::new(platform_macos::register_tools());+ reg.init_self_weak();+ let sp = socket.unwrap_or_else(serve::default_socket_path);+ let pid_path = serve::default_pid_file_path();+ serve::run_serve_cmd(reg, &sp, Some(&pid_path));+ return;+ }+ cli::Command::Stop { socket } => {+ let sp = socket.unwrap_or_else(serve::default_socket_path);+ serve::run_stop_cmd(&sp);+ return;+ }+ cli::Command::Status { socket } => {+ let sp = socket.unwrap_or_else(serve::default_socket_path);+ let pid_path = serve::default_pid_file_path();+ serve::run_status_cmd(&sp, &pid_path);+ return;+ }+ cli::Command::Recording { subcommand, args, socket } => {+ cli::run_recording_cmd(&subcommand, &args, socket.as_deref());+ return;+ }+ cli::Command::DumpDocs { pretty, doc_type } => {+ let reg = Arc::new(platform_macos::register_tools());+ cli::run_dump_docs_with_type(®, pretty, &doc_type);+ return;+ }+ cli::Command::Update { apply } => {+ cli::run_update_cmd(apply);+ return;+ }+ cli::Command::Doctor => {+ cli::run_doctor_cmd();+ return;+ }+ cli::Command::Diagnose => {+ let reg = Arc::new(platform_macos::register_tools());+ cli::run_diagnose_cmd(reg);+ return;+ }+ cli::Command::Config { subcommand, key, value, socket } => {+ let reg = Arc::new(platform_macos::register_tools());+ reg.init_self_weak();+ cli::run_config_cmd(reg, subcommand.as_deref(), key.as_deref(), value.as_deref(), socket.as_deref());+ return;+ }+ cli::Command::Mcp => {} // fall through to MCP server startup below+ }++ let cursor_cfg = cursor_overlay::CursorConfig::from_args();++ tracing::info!(+ version = env!("CARGO_PKG_VERSION"),+ cursor_id = %cursor_cfg.cursor_id,+ overlay_enabled = cursor_cfg.enabled,+ has_custom_icon = cursor_cfg.shape.is_some(),+ "cua-driver-rs starting (macOS)"+ );++ let enabled = cursor_cfg.enabled;++ // Initialise overlay channel synchronously BEFORE spawning background+ // thread. This eliminates a race where run_on_main_thread() could be+ // called before init() and find an empty channel.+ if enabled {+ platform_macos::cursor::overlay::init(cursor_cfg.clone());+ }++ // Spawn tokio + MCP server on a background thread so the main thread+ // is free for AppKit.+ // Register screenshot callback for recording (post-action screenshots).+ mcp_server::recording::set_screenshot_fn(|window_id, pid| {+ if let Some(wid) = window_id {+ platform_macos::capture::screenshot_window_bytes(wid as u32).ok()+ } else if let Some(p) = pid {+ platform_macos::windows::resolve_main_window_id(p as i32).ok()+ .and_then(|wid| platform_macos::capture::screenshot_window_bytes(wid).ok())+ } else {+ platform_macos::capture::screenshot_display_bytes().ok()+ }+ });++ // Register click-marker callback for recording (click.png with red crosshair).+ mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| {+ platform_macos::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()+ });++ std::thread::Builder::new()+ .name("cua-mcp".into())+ .spawn(move || {+ let rt = tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()+ .expect("tokio runtime");+ rt.block_on(async move {+ // Register tools; overlay init has already happened above.+ let registry = Arc::new(platform_macos::register_tools());+ // Wire up replay tool's back-reference to the registry.+ registry.init_self_weak();+ if let Err(e) = mcp_server::server::run(registry).await {+ tracing::error!("MCP server error: {e}");+ }+ });+ // MCP server exited (stdin closed / client disconnected).+ // The main thread is blocked in NSApplication.run() and won't+ // exit on its own — force-exit the process cleanly.+ std::process::exit(0);+ })+ .expect("spawn mcp thread");++ // Main thread: AppKit overlay (blocks until the process exits).+ if enabled {+ platform_macos::cursor::overlay::run_on_main_thread();+ }+ // Overlay disabled: park the main thread while the MCP background thread+ // keeps running.+ loop { std::thread::park(); }+}++// ── Non-macOS entry-point ─────────────────────────────────────────────────++#[cfg(not(target_os = "macos"))]+fn main() -> anyhow::Result<()> {+ init_logging();++ // ── CLI subcommand dispatch ──────────────────────────────────────────────+ // These commands create their own tokio runtimes internally, so they must+ // run on a plain OS thread — not inside a #[tokio::main] context which+ // would cause nested block_on panics.+ match cli::parse_command() {+ cli::Command::ListTools => {+ let reg = Arc::new(build_registry_no_cursor());+ cli::run_list_tools(®);+ return Ok(());+ }+ cli::Command::Describe(name) => {+ let reg = Arc::new(build_registry_no_cursor());+ cli::run_describe(®, &name);+ return Ok(());+ }+ cli::Command::McpConfig { client } => {+ cli::run_mcp_config(client.as_deref());+ return Ok(());+ }+ cli::Command::Call { tool, json_args, screenshot_out_file } => {+ let reg = Arc::new(build_registry_no_cursor());+ reg.init_self_weak();+ // run_call builds its own tokio runtime; must run on a fresh thread.+ std::thread::spawn(move || {+ cli::run_call(reg, &tool, json_args, screenshot_out_file);+ }).join().ok();+ return Ok(());+ }+ cli::Command::Serve { socket } => {+ // Serve mode needs the cursor overlay just like MCP mode.+ let cursor_cfg = cursor_overlay::CursorConfig::from_args();+ let reg = Arc::new(build_registry(cursor_cfg));+ reg.init_self_weak();+ let sp = socket.unwrap_or_else(serve::default_socket_path);+ let pid_path = serve::default_pid_file_path();+ // run_serve_cmd builds its own runtime; must run on a fresh thread.+ std::thread::spawn(move || {+ serve::run_serve_cmd(reg, &sp, Some(&pid_path));+ }).join().ok();+ return Ok(());+ }+ cli::Command::Stop { socket } => {+ let sp = socket.unwrap_or_else(serve::default_socket_path);+ serve::run_stop_cmd(&sp);+ return Ok(());+ }+ cli::Command::Status { socket } => {+ let sp = socket.unwrap_or_else(serve::default_socket_path);+ let pid_path = serve::default_pid_file_path();+ serve::run_status_cmd(&sp, &pid_path);+ return Ok(());+ }+ cli::Command::Recording { subcommand, args, socket } => {+ cli::run_recording_cmd(&subcommand, &args, socket.as_deref());+ return Ok(());+ }+ cli::Command::DumpDocs { pretty, doc_type } => {+ let reg = Arc::new(build_registry_no_cursor());+ cli::run_dump_docs_with_type(®, pretty, &doc_type);+ return Ok(());+ }+ cli::Command::Update { apply } => {+ cli::run_update_cmd(apply);+ return Ok(());+ }+ cli::Command::Doctor => {+ cli::run_doctor_cmd();+ return Ok(());+ }+ cli::Command::Diagnose => {+ let reg = Arc::new(build_registry_no_cursor());+ cli::run_diagnose_cmd(reg);+ return Ok(());+ }+ cli::Command::Config { subcommand, key, value, socket } => {+ let reg = Arc::new(build_registry_no_cursor());+ reg.init_self_weak();+ std::thread::spawn(move || {+ cli::run_config_cmd(reg, subcommand.as_deref(), key.as_deref(), value.as_deref(), socket.as_deref());+ }).join().ok();+ return Ok(());+ }+ cli::Command::Mcp => {} // fall through to MCP server startup below+ }++ // MCP server mode: this needs a full async tokio runtime.+ let rt = tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()+ .expect("tokio runtime");+ rt.block_on(async_main())?;+ Ok(())+}++#[cfg(not(target_os = "macos"))]+async fn async_main() -> anyhow::Result<()> {++ let cursor_cfg = cursor_overlay::CursorConfig::from_args();++ tracing::info!(+ version = env!("CARGO_PKG_VERSION"),+ os = std::env::consts::OS,+ cursor_id = %cursor_cfg.cursor_id,+ overlay_enabled = cursor_cfg.enabled,+ has_custom_icon = cursor_cfg.shape.is_some(),+ "cua-driver-rs starting"+ );++ let registry = Arc::new(build_registry(cursor_cfg));+ registry.init_self_weak();+ mcp_server::server::run(registry).await?;+ Ok(())+}++// ── Registry builder (non-macOS) ──────────────────────────────────────────++#[cfg(not(target_os = "macos"))]+fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> mcp_server::tool::ToolRegistry {+ #[cfg(target_os = "windows")]+ {+ mcp_server::recording::set_screenshot_fn(|window_id, pid| {+ if let Some(hwnd) = window_id {+ platform_windows::capture::screenshot_window_bytes(hwnd).ok()+ } else if let Some(p) = pid {+ let wins = platform_windows::win32::list_windows(Some(p as u32));+ wins.first().and_then(|w| {+ platform_windows::capture::screenshot_window_bytes(w.hwnd).ok()+ })+ } else {+ platform_windows::capture::screenshot_display_bytes().ok()+ }+ });+ mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| {+ platform_windows::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()+ });+ platform_windows::register_tools_with_cursor(cursor_cfg)+ }+ #[cfg(target_os = "linux")]+ {+ mcp_server::recording::set_screenshot_fn(|window_id, pid| {+ if let Some(xid) = window_id {+ platform_linux::capture::screenshot_window_bytes(xid).ok()+ } else if let Some(p) = pid {+ let wins = platform_linux::x11::list_windows(Some(p as u32));+ wins.first().and_then(|w| {+ platform_linux::capture::screenshot_window_bytes(w.xid).ok()+ })+ } else {+ platform_linux::capture::screenshot_display_bytes().ok()+ }+ });+ mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| {+ platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()+ });+ platform_linux::register_tools_with_cursor(cursor_cfg)+ }+ #[cfg(not(any(target_os = "windows", target_os = "linux")))]+ {+ let _ = cursor_cfg;+ let mut r = mcp_server::tool::ToolRegistry::new();+ r.register(Box::new(crate::stub::UnsupportedPlatformTool));+ r+ }+}++/// Build a registry without initialising the cursor overlay.+/// Used by CLI subcommands (list-tools / describe / call) that don't need the overlay.+#[cfg(not(target_os = "macos"))]+fn build_registry_no_cursor() -> mcp_server::tool::ToolRegistry {+ #[cfg(target_os = "windows")]+ {+ mcp_server::recording::set_screenshot_fn(|window_id, pid| {+ if let Some(hwnd) = window_id {+ platform_windows::capture::screenshot_window_bytes(hwnd).ok()+ } else if let Some(p) = pid {+ let wins = platform_windows::win32::list_windows(Some(p as u32));+ wins.first().and_then(|w| {+ platform_windows::capture::screenshot_window_bytes(w.hwnd).ok()+ })+ } else {+ platform_windows::capture::screenshot_display_bytes().ok()+ }+ });+ mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| {+ platform_windows::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()+ });+ platform_windows::register_tools_with_cursor(cursor_overlay::CursorConfig { enabled: false, ..Default::default() })+ }+ #[cfg(target_os = "linux")]+ {+ mcp_server::recording::set_screenshot_fn(|window_id, pid| {+ if let Some(xid) = window_id {+ platform_linux::capture::screenshot_window_bytes(xid).ok()+ } else if let Some(p) = pid {+ let wins = platform_linux::x11::list_windows(Some(p as u32));+ wins.first().and_then(|w| {+ platform_linux::capture::screenshot_window_bytes(w.xid).ok()+ })+ } else {+ platform_linux::capture::screenshot_display_bytes().ok()+ }+ });+ mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| {+ platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()+ });+ platform_linux::register_tools_with_cursor(cursor_overlay::CursorConfig { enabled: false, ..Default::default() })+ }+ #[cfg(not(any(target_os = "windows", target_os = "linux")))]+ {+ let mut r = mcp_server::tool::ToolRegistry::new();+ r.register(Box::new(crate::stub::UnsupportedPlatformTool));+ r+ }+}++#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]+mod stub {+ use async_trait::async_trait;+ use mcp_server::tool::{Tool, ToolDef, ToolResult};+ use serde_json::Value;++ pub struct UnsupportedPlatformTool;++ #[async_trait]+ impl Tool for UnsupportedPlatformTool {+ fn def(&self) -> &ToolDef {+ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+ DEF.get_or_init(|| ToolDef {+ name: "unsupported_platform".into(),+ description: "This platform is not supported.".into(),+ input_schema: serde_json::json!({"type":"object","properties":{}}),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ ToolResult::error("Unsupported platform")+ }+ }+}diff --git a/libs/cua-driver-rs/crates/cua-driver/src/serve.rs b/libs/cua-driver-rs/crates/cua-driver/src/serve.rsnew file mode 100644index 0000000000..34a69a8886--- /dev/null+++ b/libs/cua-driver-rs/crates/cua-driver/src/serve.rs@@ -0,0 +1,628 @@+//! Unix-socket daemon server and client for `cua-driver serve`/`stop`/`status`.+//!+//! Protocol: line-delimited JSON over a Unix domain socket.+//!+//! Request shapes:+//! {"method":"call","name":"<tool>","args":{...}}+//! {"method":"list"}+//! {"method":"describe","name":"<tool>"}+//! {"method":"shutdown"}+//!+//! Response shapes:+//! {"ok":true,"result":...}+//! {"ok":false,"error":"...","exit_code":1}+//!+//! The socket file is at:+//! macOS — ~/Library/Caches/cua-driver/cua-driver.sock+//! Linux — ~/.cache/cua-driver/cua-driver.sock+//! Windows — \\.\pipe\cua-driver (TODO: use named pipe; stubs only for now)++use serde::{Deserialize, Serialize};++// ── Paths ─────────────────────────────────────────────────────────────────────++/// Returns the platform default socket/pipe path.+pub fn default_socket_path() -> String {+ #[cfg(target_os = "macos")]+ {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ format!("{home}/Library/Caches/cua-driver/cua-driver.sock")+ }+ #[cfg(target_os = "linux")]+ {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ format!("{home}/.cache/cua-driver/cua-driver.sock")+ }+ #[cfg(target_os = "windows")]+ {+ r"\\.\pipe\cua-driver".to_owned()+ }+ #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]+ {+ "/tmp/cua-driver.sock".to_owned()+ }+}++/// Returns the platform default PID file path.+pub fn default_pid_file_path() -> String {+ #[cfg(target_os = "macos")]+ {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ format!("{home}/Library/Caches/cua-driver/cua-driver.pid")+ }+ #[cfg(target_os = "linux")]+ {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ format!("{home}/.cache/cua-driver/cua-driver.pid")+ }+ #[cfg(target_os = "windows")]+ {+ let local = std::env::var("LOCALAPPDATA").unwrap_or_else(|_| "C:/Temp".into());+ format!("{local}/cua-driver/cua-driver.pid")+ }+ #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]+ {+ "/tmp/cua-driver.pid".to_owned()+ }+}++// ── Protocol types ────────────────────────────────────────────────────────────++#[derive(Debug, Serialize, Deserialize)]+pub struct DaemonRequest {+ pub method: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub name: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub args: Option<serde_json::Value>,+}++#[derive(Debug, Serialize, Deserialize)]+pub struct DaemonResponse {+ pub ok: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub result: Option<serde_json::Value>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub exit_code: Option<i32>,+}++impl DaemonResponse {+ pub fn ok(result: serde_json::Value) -> Self {+ Self { ok: true, result: Some(result), error: None, exit_code: None }+ }+ pub fn err(msg: impl Into<String>, code: i32) -> Self {+ Self { ok: false, result: None, error: Some(msg.into()), exit_code: Some(code) }+ }+}++// ── Client ────────────────────────────────────────────────────────────────────++/// Probe whether a daemon is listening on `socket_path` by sending a+/// trivial `list` request and checking for a valid response.+pub fn is_daemon_listening(socket_path: &str) -> bool {+ let req = DaemonRequest { method: "list".into(), name: None, args: None };+ send_request(socket_path, &req)+ .ok()+ .map(|r| r.ok)+ .unwrap_or(false)+}++/// Read the PID stored in `pid_file_path`, if any.+pub fn read_pid_file(pid_file_path: &str) -> Option<u32> {+ std::fs::read_to_string(pid_file_path)+ .ok()+ .and_then(|s| s.trim().parse().ok())+}++/// Send a request to the daemon and return the response.+/// Uses a 3-second connect timeout (by polling) and a 10-second read timeout.+#[cfg(unix)]+pub fn send_request(socket_path: &str, req: &DaemonRequest) -> anyhow::Result<DaemonResponse> {+ use std::io::{BufRead, BufReader, Write};+ use std::os::unix::net::UnixStream;+ use std::time::Duration;++ let stream = UnixStream::connect(socket_path)+ .map_err(|e| anyhow::anyhow!("connect to {socket_path}: {e}"))?;+ stream.set_write_timeout(Some(Duration::from_secs(5)))?;+ stream.set_read_timeout(Some(Duration::from_secs(10)))?;++ let mut w = stream.try_clone()?;+ let line = serde_json::to_string(req)? + "\n";+ w.write_all(line.as_bytes())?;+ w.flush()?;++ let reader = BufReader::new(stream);+ let mut resp_line = String::new();+ reader.lines().next()+ .ok_or_else(|| anyhow::anyhow!("daemon closed connection without response"))??+ .clone_into(&mut resp_line);+ let resp: DaemonResponse = serde_json::from_str(&resp_line)?;+ Ok(resp)+}++#[cfg(not(unix))]+pub fn send_request(socket_path: &str, req: &DaemonRequest) -> anyhow::Result<DaemonResponse> {+ #[cfg(target_os = "windows")]+ {+ use std::io::{BufRead, BufReader, Write};+ use std::time::Duration;++ // Retry opening the pipe — server may still be starting.+ let deadline = std::time::Instant::now() + Duration::from_secs(3);+ let pipe = loop {+ match std::fs::OpenOptions::new()+ .read(true)+ .write(true)+ .open(socket_path)+ {+ Ok(f) => break f,+ Err(_) if std::time::Instant::now() < deadline => {+ std::thread::sleep(Duration::from_millis(50));+ }+ Err(e) => anyhow::bail!("connect to named pipe {socket_path}: {e}"),+ }+ };++ let mut writer = pipe.try_clone()?;+ let line = serde_json::to_string(req)? + "\n";+ writer.write_all(line.as_bytes())?;+ writer.flush()?;++ // Server writes one JSON line then flushes; pipe stays open until we drop it.+ let reader = BufReader::new(pipe);+ let resp_line = reader+ .lines()+ .next()+ .ok_or_else(|| anyhow::anyhow!("daemon closed connection without response"))??;+ let resp: DaemonResponse = serde_json::from_str(&resp_line)?;+ Ok(resp)+ }+ #[cfg(not(target_os = "windows"))]+ {+ anyhow::bail!("daemon client not supported on this platform (socket path: {socket_path})");+ }+}++// ── Server ────────────────────────────────────────────────────────────────────++/// Run the daemon server. Binds `socket_path`, writes `pid_file_path`,+/// accepts connections, and serves requests until `{"method":"shutdown"}`.+///+/// This is `async` and must be called from a tokio runtime.+#[cfg(unix)]+pub async fn run_serve(+ registry: std::sync::Arc<mcp_server::tool::ToolRegistry>,+ socket_path: &str,+ pid_file_path: Option<&str>,+) -> anyhow::Result<()> {+ use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};+ use tokio::net::UnixListener;++ // Create parent directory.+ if let Some(dir) = std::path::Path::new(socket_path).parent() {+ std::fs::create_dir_all(dir)?;+ }++ // Remove stale socket file (from a crashed previous daemon).+ let _ = std::fs::remove_file(socket_path);++ let listener = UnixListener::bind(socket_path)+ .map_err(|e| anyhow::anyhow!("bind {socket_path}: {e}"))?;++ eprintln!("cua-driver daemon listening on {socket_path}");++ // Write PID file.+ if let Some(pid_path) = pid_file_path {+ if let Some(dir) = std::path::Path::new(pid_path).parent() {+ let _ = std::fs::create_dir_all(dir);+ }+ let _ = std::fs::write(pid_path, std::process::id().to_string());+ }++ // Shutdown channel.+ let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>();+ let shutdown_tx = std::sync::Arc::new(tokio::sync::Mutex::new(Some(shutdown_tx)));++ loop {+ tokio::select! {+ result = listener.accept() => {+ let (stream, _) = result?;+ let reg = registry.clone();+ let shutdown_tx2 = shutdown_tx.clone();++ tokio::spawn(async move {+ let (reader, mut writer) = stream.into_split();+ let mut lines = BufReader::new(reader).lines();++ while let Ok(Some(line)) = lines.next_line().await {+ let req: DaemonRequest = match serde_json::from_str(&line) {+ Ok(r) => r,+ Err(e) => {+ let resp = DaemonResponse::err(+ format!("JSON parse error: {e}"), 65+ );+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ continue;+ }+ };++ match req.method.as_str() {+ "shutdown" => {+ let resp = DaemonResponse::ok(serde_json::json!({"shutdown": true}));+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ let mut guard = shutdown_tx2.lock().await;+ if let Some(tx) = guard.take() {+ let _ = tx.send(());+ }+ return;+ }+ "list" => {+ let tools: Vec<serde_json::Value> = reg.iter_defs()+ .map(|(name, def)| serde_json::json!({+ "name": name,+ "description": def.description+ }))+ .collect();+ let resp = DaemonResponse::ok(serde_json::json!({"tools": tools}));+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ "describe" => {+ let name = req.name.as_deref().unwrap_or("");+ match reg.get_def(name) {+ Some(def) => {+ let resp = DaemonResponse::ok(serde_json::json!({+ "name": def.name,+ "description": def.description,+ "input_schema": def.input_schema+ }));+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ None => {+ let resp = DaemonResponse::err(+ format!("Unknown tool: {name}"), 64+ );+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ }+ }+ "call" => {+ let raw_name = req.name.as_deref().unwrap_or("").to_owned();+ // Deprecated alias: `type_text_chars` → `type_text`.+ // Mirrors Swift's `ToolRegistry.call` aliasing.+ let tool_name = if raw_name == "type_text_chars" {+ eprintln!("[cua-driver-rs] deprecated tool name 'type_text_chars' — use 'type_text' instead.");+ "type_text".to_owned()+ } else { raw_name.clone() };+ let args = req.args.unwrap_or(serde_json::Value::Object(+ serde_json::Map::new()+ ));+ if reg.get_def(&tool_name).is_none() {+ let resp = DaemonResponse::err(+ format!("Unknown tool: {tool_name}"), 64+ );+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ continue;+ }+ let result = reg.invoke(&tool_name, args).await;+ let is_err = result.is_error.unwrap_or(false);+ let content: Vec<serde_json::Value> = result.content.iter().map(|c| {+ match c {+ mcp_server::protocol::Content::Text { text, .. } =>+ serde_json::json!({"type":"text","text":text}),+ mcp_server::protocol::Content::Image { data, mime_type, .. } =>+ serde_json::json!({"type":"image","data":data,"mimeType":mime_type}),+ }+ }).collect();+ let mut result_obj = serde_json::json!({+ "content": content,+ "isError": is_err+ });+ if let Some(sc) = result.structured_content {+ result_obj["structuredContent"] = sc;+ }+ let resp = if is_err {+ DaemonResponse::err(+ result.content.iter()+ .filter_map(|c| if let mcp_server::protocol::Content::Text { text, .. } = c { Some(text.as_str()) } else { None })+ .collect::<Vec<_>>()+ .join("\n"),+ 1+ )+ } else {+ DaemonResponse::ok(result_obj)+ };+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ other => {+ let resp = DaemonResponse::err(+ format!("Unknown method: {other}"), 65+ );+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ }+ }+ });+ }+ _ = &mut shutdown_rx => {+ eprintln!("cua-driver daemon shutting down.");+ break;+ }+ }+ }++ // Clean up.+ let _ = std::fs::remove_file(socket_path);+ if let Some(pid_path) = pid_file_path {+ let _ = std::fs::remove_file(pid_path);+ }++ Ok(())+}++#[cfg(target_os = "windows")]+pub async fn run_serve(+ registry: std::sync::Arc<mcp_server::tool::ToolRegistry>,+ socket_path: &str,+ pid_file_path: Option<&str>,+) -> anyhow::Result<()> {+ use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};+ use tokio::net::windows::named_pipe::ServerOptions;++ eprintln!("cua-driver daemon listening on {socket_path}");++ // Write PID file.+ if let Some(pid_path) = pid_file_path {+ if let Some(dir) = std::path::Path::new(pid_path).parent() {+ let _ = std::fs::create_dir_all(dir);+ }+ let _ = std::fs::write(pid_path, std::process::id().to_string());+ }++ let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>();+ let shutdown_tx = std::sync::Arc::new(tokio::sync::Mutex::new(Some(shutdown_tx)));++ loop {+ // Create a new pipe server instance to accept the next client.+ let server = ServerOptions::new()+ .first_pipe_instance(false)+ .create(socket_path)+ .map_err(|e| anyhow::anyhow!("create named pipe {socket_path}: {e}"))?;++ tokio::select! {+ result = server.connect() => {+ result.map_err(|e| anyhow::anyhow!("named pipe connect: {e}"))?;++ let reg = registry.clone();+ let shutdown_tx2 = shutdown_tx.clone();++ tokio::spawn(async move {+ let (reader, mut writer) = tokio::io::split(server);+ let mut lines = BufReader::new(reader).lines();++ while let Ok(Some(line)) = lines.next_line().await {+ let req: DaemonRequest = match serde_json::from_str(&line) {+ Ok(r) => r,+ Err(e) => {+ let resp = DaemonResponse::err(format!("JSON parse error: {e}"), 65);+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ continue;+ }+ };++ match req.method.as_str() {+ "shutdown" => {+ let resp = DaemonResponse::ok(serde_json::json!({"shutdown": true}));+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ let mut guard = shutdown_tx2.lock().await;+ if let Some(tx) = guard.take() { let _ = tx.send(()); }+ return;+ }+ "list" => {+ let tools: Vec<serde_json::Value> = reg.iter_defs()+ .map(|(name, def)| serde_json::json!({"name": name, "description": def.description}))+ .collect();+ let resp = DaemonResponse::ok(serde_json::json!({"tools": tools}));+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ "describe" => {+ let name = req.name.as_deref().unwrap_or("");+ let resp = match reg.get_def(name) {+ Some(def) => DaemonResponse::ok(serde_json::json!({+ "name": def.name, "description": def.description,+ "input_schema": def.input_schema+ })),+ None => DaemonResponse::err(format!("Unknown tool: {name}"), 64),+ };+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ "call" => {+ let raw_name = req.name.as_deref().unwrap_or("").to_owned();+ let tool_name = if raw_name == "type_text_chars" {+ eprintln!("[cua-driver-rs] deprecated tool name 'type_text_chars' — use 'type_text' instead.");+ "type_text".to_owned()+ } else { raw_name.clone() };+ let args = req.args.unwrap_or(serde_json::Value::Object(serde_json::Map::new()));+ if reg.get_def(&tool_name).is_none() {+ let resp = DaemonResponse::err(format!("Unknown tool: {tool_name}"), 64);+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ continue;+ }+ let result = reg.invoke(&tool_name, args).await;+ let is_err = result.is_error.unwrap_or(false);+ let content: Vec<serde_json::Value> = result.content.iter().map(|c| match c {+ mcp_server::protocol::Content::Text { text, .. } =>+ serde_json::json!({"type":"text","text":text}),+ mcp_server::protocol::Content::Image { data, mime_type, .. } =>+ serde_json::json!({"type":"image","data":data,"mimeType":mime_type}),+ }).collect();+ let mut result_obj = serde_json::json!({"content": content, "isError": is_err});+ if let Some(sc) = result.structured_content {+ result_obj["structuredContent"] = sc;+ }+ let resp = if is_err {+ DaemonResponse::err(+ result.content.iter()+ .filter_map(|c| if let mcp_server::protocol::Content::Text { text, .. } = c { Some(text.as_str()) } else { None })+ .collect::<Vec<_>>().join("\n"),+ 1+ )+ } else {+ DaemonResponse::ok(result_obj)+ };+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ other => {+ let resp = DaemonResponse::err(format!("Unknown method: {other}"), 65);+ let _ = writer.write_all(+ (serde_json::to_string(&resp).unwrap() + "\n").as_bytes()+ ).await;+ }+ }+ }+ });+ }+ _ = &mut shutdown_rx => {+ eprintln!("cua-driver daemon shutting down.");+ break;+ }+ }+ }++ if let Some(pid_path) = pid_file_path {+ let _ = std::fs::remove_file(pid_path);+ }+ Ok(())+}++#[cfg(not(any(unix, target_os = "windows")))]+pub async fn run_serve(+ _registry: std::sync::Arc<mcp_server::tool::ToolRegistry>,+ _socket_path: &str,+ _pid_file_path: Option<&str>,+) -> anyhow::Result<()> {+ anyhow::bail!("cua-driver serve is not supported on this platform");+}++// ── CLI helpers ───────────────────────────────────────────────────────────────++/// `cua-driver serve` implementation.+pub fn run_serve_cmd(+ registry: std::sync::Arc<mcp_server::tool::ToolRegistry>,+ socket_path: &str,+ pid_file_path: Option<&str>,+) {+ let socket_path = socket_path.to_owned();+ let pid_file_path = pid_file_path.map(str::to_owned);++ // Fail fast if another daemon is already running.+ if is_daemon_listening(&socket_path) {+ let pid_hint = pid_file_path.as_deref()+ .and_then(|p| read_pid_file(p))+ .map(|pid| format!(" (pid {pid})"))+ .unwrap_or_default();+ eprintln!(+ "cua-driver daemon is already running on {socket_path}{pid_hint}. \+ Run `cua-driver stop` first."+ );+ std::process::exit(1);+ }++ let rt = tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()+ .expect("tokio runtime");++ if let Err(e) = rt.block_on(run_serve(+ registry,+ &socket_path,+ pid_file_path.as_deref(),+ )) {+ eprintln!("cua-driver serve error: {e}");+ std::process::exit(1);+ }+}++/// `cua-driver stop` implementation.+pub fn run_stop_cmd(socket_path: &str) {+ if !is_daemon_listening(socket_path) {+ eprintln!("cua-driver daemon is not running");+ std::process::exit(1);+ }++ let req = DaemonRequest { method: "shutdown".into(), name: None, args: None };+ match send_request(socket_path, &req) {+ Ok(_) => {+ // Poll until daemon stops responding (up to 2 seconds).+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);+ loop {+ // On Windows the pipe path isn't a filesystem file; probe liveness instead.+ let gone = if std::path::Path::new(socket_path).exists() {+ false+ } else {+ !is_daemon_listening(socket_path)+ };+ if gone {+ // Swift's `stop` exits silently on success (no stdout+ // line) — match that for byte-for-byte parity.+ return;+ }+ if std::time::Instant::now() >= deadline {+ eprintln!("cua-driver daemon did not release socket within 2s");+ std::process::exit(1);+ }+ std::thread::sleep(std::time::Duration::from_millis(50));+ }+ }+ Err(e) => {+ eprintln!("stop: {e}");+ std::process::exit(1);+ }+ }+}++/// `cua-driver status` implementation.+pub fn run_status_cmd(socket_path: &str, pid_file_path: &str) {+ if is_daemon_listening(socket_path) {+ println!("cua-driver daemon is running");+ println!(" socket: {socket_path}");+ if let Some(pid) = read_pid_file(pid_file_path) {+ println!(" pid: {pid}");+ } else {+ println!(" pid: unknown (no pid file)");+ }+ } else {+ eprintln!("cua-driver daemon is not running");+ std::process::exit(1);+ }+}diff --git a/libs/cua-driver-rs/crates/cua-driver/tests/focus_check_test.rs b/libs/cua-driver-rs/crates/cua-driver/tests/focus_check_test.rsnew file mode 100644index 0000000000..bd2d4ea5b9--- /dev/null+++ b/libs/cua-driver-rs/crates/cua-driver/tests/focus_check_test.rs@@ -0,0 +1,232 @@+//! Integration test: verify that background automation does NOT steal focus.+//!+//! Test strategy:+//! 1. Open a "focus check window" (Terminal) and confirm it is focused.+//! 2. Start a cua-driver-rs MCP server process.+//! 3. Send automation actions (click, type_text) to Calculator (a different app).+//! 4. After each action, verify the Terminal window is STILL the active window.+//!+//! This test requires:+//! - macOS (or Linux/Windows equivalents)+//! - Calculator app installed (com.apple.calculator on macOS)+//! - Accessibility permission granted+//!+//! Run with: cargo test --test focus_check_test -- --nocapture+//! Requires: `cargo build` first to produce the binary.++use std::io::{BufRead, BufReader, Write};+use std::process::{Command, Stdio};+use std::time::Duration;+use std::thread;++#[cfg(target_os = "macos")]+mod macos_focus_tests {+ use super::*;++ fn binary_path() -> std::path::PathBuf {+ let manifest = std::env::var("CARGO_MANIFEST_DIR")+ .expect("CARGO_MANIFEST_DIR not set");+ std::path::PathBuf::from(manifest)+ .parent().unwrap() // crates/+ .parent().unwrap() // workspace root+ .join("target/debug/cua-driver")+ }++ fn send(stdin: &mut impl Write, req: serde_json::Value) {+ writeln!(stdin, "{}", serde_json::to_string(&req).unwrap()).unwrap();+ }++ fn recv(stdout: &mut impl BufRead) -> serde_json::Value {+ let mut line = String::new();+ stdout.read_line(&mut line).unwrap();+ serde_json::from_str(line.trim()).unwrap()+ }++ /// Get the bundle ID of the current frontmost app via osascript.+ fn frontmost_bundle_id() -> String {+ let out = Command::new("osascript")+ .arg("-e")+ .arg(r#"tell application "System Events" to bundle identifier of first application process whose frontmost is true"#)+ .output()+ .expect("osascript");+ String::from_utf8_lossy(&out.stdout).trim().to_owned()+ }++ /// Bring Terminal to the front (our "focus check window").+ fn focus_terminal() {+ let _ = Command::new("osascript")+ .arg("-e")+ .arg(r#"tell application "Terminal" to activate"#)+ .output();+ thread::sleep(Duration::from_millis(300));+ }++ /// Open Calculator in background.+ fn open_calculator_background() {+ let _ = Command::new("open")+ .args(["-g", "-b", "com.apple.calculator"])+ .output();+ thread::sleep(Duration::from_secs(1));+ }++ fn find_calculator_pid(stdout: &mut impl BufRead, stdin: &mut impl Write) -> Option<i32> {+ send(stdin, serde_json::json!({+ "jsonrpc": "2.0", "id": 100, "method": "tools/call",+ "params": { "name": "list_apps", "arguments": {} }+ }));+ let resp = recv(stdout);+ // Use structuredContent.apps array (preferred over text parsing).+ if let Some(apps) = resp["result"]["structuredContent"]["apps"].as_array() {+ for app in apps {+ let bundle = app["bundle_id"].as_str().unwrap_or("");+ let name = app["name"].as_str().unwrap_or("");+ if bundle.contains("calculator") || name.eq_ignore_ascii_case("calculator") {+ if let Some(pid) = app["pid"].as_i64() {+ return Some(pid as i32);+ }+ }+ }+ }+ // Fallback: parse text content for older binary versions.+ let text = resp["result"]["content"][0]["text"].as_str()?;+ for line in text.lines() {+ if line.contains("com.apple.calculator") || line.contains("Calculator") {+ if let Some(pid_str) = line.split("(pid ").nth(1)+ .and_then(|s| s.split(')').next())+ {+ return pid_str.trim().parse().ok();+ }+ }+ }+ None+ }++ #[test]+ #[ignore] // Run explicitly: cargo test --test focus_check_test focus_not_stolen -- --ignored --nocapture+ fn focus_not_stolen_during_calculator_click() {+ let binary = binary_path();+ if !binary.exists() {+ eprintln!("Binary not built. Run `cargo build` first.");+ return;+ }++ // Setup: open Calculator in background, bring Terminal to front.+ open_calculator_background();+ focus_terminal();++ let initial_focus = frontmost_bundle_id();+ println!("Initial focus: {}", initial_focus);+ // Terminal or our test runner should be focused.++ // Start the MCP driver.+ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn cua-driver-rs");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ // Initialize.+ send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ recv(&mut stdout);++ // Find Calculator.+ let calc_pid = find_calculator_pid(&mut stdout, stdin);+ let calc_pid = match calc_pid {+ Some(p) => p,+ None => {+ eprintln!("Calculator not found in running apps");+ child.kill().ok();+ return;+ }+ };+ println!("Calculator pid: {}", calc_pid);++ // Get Calculator's window.+ send(stdin, serde_json::json!({+ "jsonrpc": "2.0", "id": 200, "method": "tools/call",+ "params": {+ "name": "list_windows",+ "arguments": { "pid": calc_pid }+ }+ }));+ let resp = recv(&mut stdout);+ let windows = resp["result"]["structuredContent"]["windows"].as_array()+ .expect("windows array");+ if windows.is_empty() {+ eprintln!("No windows for Calculator");+ child.kill().ok();+ return;+ }+ let window_id = windows[0]["window_id"].as_u64().unwrap() as u32;+ println!("Calculator window_id: {}", window_id);++ // Walk AX tree.+ send(stdin, serde_json::json!({+ "jsonrpc": "2.0", "id": 300, "method": "tools/call",+ "params": {+ "name": "get_window_state",+ "arguments": { "pid": calc_pid, "window_id": window_id, "capture_mode": "tree" }+ }+ }));+ let resp = recv(&mut stdout);+ println!("get_window_state: {}", resp["result"]["content"][0]["text"].as_str().unwrap_or("(none)").chars().take(200).collect::<String>());++ // Verify focus hasn't been stolen yet.+ let focus_after_get = frontmost_bundle_id();+ println!("Focus after get_window_state: {}", focus_after_get);+ assert_eq!(+ focus_after_get, initial_focus,+ "get_window_state STOLE FOCUS! Was: {}, now: {}", initial_focus, focus_after_get+ );++ // Find a button element to click (element_index 1 is the Delete/AC button+ // in Calculator's AX tree and supports AXPress).+ // We use element_index=1 because [0] is the AXWindow itself (only raises).+ send(stdin, serde_json::json!({+ "jsonrpc": "2.0", "id": 400, "method": "tools/call",+ "params": {+ "name": "click",+ "arguments": { "pid": calc_pid, "window_id": window_id, "element_index": 1 }+ }+ }));+ let resp = recv(&mut stdout);+ let click_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("?");+ println!("click result: {}", click_text);+ // Accept either a successful click or a supported AX error — what matters is no crash/focus steal.++ thread::sleep(Duration::from_millis(200));++ // CRITICAL: verify focus was not stolen.+ let focus_after_click = frontmost_bundle_id();+ println!("Focus after click: {}", focus_after_click);+ assert_eq!(+ focus_after_click, initial_focus,+ "click() STOLE FOCUS! Expected {} to remain focused, got {}", initial_focus, focus_after_click+ );++ // type_text also must not steal focus.+ send(stdin, serde_json::json!({+ "jsonrpc": "2.0", "id": 500, "method": "tools/call",+ "params": {+ "name": "type_text",+ "arguments": { "pid": calc_pid, "text": "5" }+ }+ }));+ let resp = recv(&mut stdout);+ println!("type_text result: {}", resp["result"]["content"][0]["text"].as_str().unwrap_or("?"));+ thread::sleep(Duration::from_millis(200));+ let focus_after_type = frontmost_bundle_id();+ println!("Focus after type_text: {}", focus_after_type);+ assert_eq!(+ focus_after_type, initial_focus,+ "type_text() STOLE FOCUS! Expected {} to remain focused, got {}", initial_focus, focus_after_type+ );++ println!("✅ Focus was NOT stolen by click or type_text. Background automation confirmed.");+ child.kill().ok();+ }+}diff --git a/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs b/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rsnew file mode 100644index 0000000000..76dd577d36--- /dev/null+++ b/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs@@ -0,0 +1,3253 @@+//! Integration tests for the MCP JSON-RPC 2.0 protocol.+//!+//! Tests the server protocol without any platform-specific automation —+//! verifies request/response format, initialize handshake, tools/list,+//! and error handling.++use std::io::{BufRead, BufReader, Write};+use std::process::{Command, Stdio};+use std::time::Duration;++/// Path to the built binary.+fn binary_path() -> std::path::PathBuf {+ // CARGO_MANIFEST_DIR is crates/cua-driver; workspace root is two levels up.+ let manifest = std::env::var("CARGO_MANIFEST_DIR")+ .expect("CARGO_MANIFEST_DIR not set");+ std::path::PathBuf::from(manifest)+ .parent().unwrap() // crates/+ .parent().unwrap() // workspace root (cua-driver-rs/)+ .join("target/debug/cua-driver")+}++fn send_request(stdin: &mut impl Write, request: &serde_json::Value) {+ let line = serde_json::to_string(request).unwrap();+ writeln!(stdin, "{}", line).unwrap();+}++fn read_response(reader: &mut impl BufRead) -> serde_json::Value {+ let mut line = String::new();+ reader.read_line(&mut line).expect("read line");+ serde_json::from_str(line.trim()).expect("parse JSON")+}++#[test]+#[cfg(target_os = "macos")]+fn test_initialize_handshake() {+ let binary = binary_path();+ if !binary.exists() {+ eprintln!("Binary not found at {:?} — run `cargo build` first", binary);+ return;+ }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn cua-driver");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ // 1. Send initialize request.+ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "id": 1,+ "method": "initialize",+ "params": {+ "protocolVersion": "2024-11-05",+ "clientInfo": { "name": "test-client", "version": "0.0.1" }+ }+ }));++ let resp = read_response(&mut stdout);+ assert_eq!(resp["jsonrpc"], "2.0");+ assert_eq!(resp["id"], 1);+ assert!(resp["result"]["protocolVersion"].is_string());+ assert_eq!(resp["result"]["serverInfo"]["name"], "cua-driver-rs");++ // 2. Send notifications/initialized (no response expected).+ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "method": "notifications/initialized"+ }));++ // 3. Request tool list.+ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "id": 2,+ "method": "tools/list"+ }));++ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 2);+ let tools = resp["result"]["tools"].as_array().expect("tools array");+ assert!(!tools.is_empty(), "Should have at least one tool");++ // Verify tool names match reference implementation.+ let tool_names: Vec<&str> = tools.iter()+ .filter_map(|t| t["name"].as_str())+ .collect();++ for expected in &["list_apps", "list_windows", "get_window_state", "click", "type_text", "press_key"] {+ assert!(+ tool_names.contains(expected),+ "Missing tool: {} (have: {:?})", expected, tool_names+ );+ }++ // 4. Verify unknown method returns -32601.+ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "id": 3,+ "method": "unknown/method"+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 3);+ assert_eq!(resp["error"]["code"], -32601);++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_tools_call_list_apps() {+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ // Initialize.+ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Call list_apps.+ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "id": 2,+ "method": "tools/call",+ "params": { "name": "list_apps", "arguments": {} }+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 2);+ assert!(resp["result"]["content"].is_array());+ let content = &resp["result"]["content"][0];+ assert_eq!(content["type"], "text");+ let text = content["text"].as_str().unwrap();+ // Should contain some running apps.+ assert!(text.contains("Found"), "Expected app list text, got: {}", text);++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_tools_call_unknown_tool() {+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "id": 2,+ "method": "tools/call",+ "params": { "name": "nonexistent_tool", "arguments": {} }+ }));+ let resp = read_response(&mut stdout);+ // Error should be in the content with isError=true, not a protocol error.+ let result = &resp["result"];+ let is_error = result["isError"].as_bool().unwrap_or(false);+ assert!(is_error, "Expected isError=true for unknown tool");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_concurrent_clients() {+ //! Verify two concurrent cua-driver-rs processes both respond correctly.+ //! This tests the "multiple cua-driver processes" scenario relevant to+ //! the multi-cursor use case.++ let binary = binary_path();+ if !binary.exists() { return; }++ let mut processes: Vec<std::process::Child> = (0..2).map(|_| {+ Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn")+ }).collect();++ for (i, child) in processes.iter_mut().enumerate() {+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "id": i + 1,+ "method": "initialize",+ "params": {}+ }));++ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], (i + 1) as i64);+ assert!(resp["result"]["protocolVersion"].is_string(),+ "Process {i} failed to initialize");+ }++ for mut child in processes {+ child.kill().ok();+ }+}++#[test]+#[cfg(target_os = "macos")]+fn test_overlay_move_cursor_stays_alive() {+ //! Verify that calling move_cursor with the overlay enabled does not crash the process.+ //! The overlay renders to a transparent NSWindow on the main thread; the MCP server+ //! sends commands via the global channel. Any crash in the render thread (e.g. wrong+ //! GCD queue pointer, bad CGImage argument types) would cause this test to fail with+ //! an early EOF or a non-zero exit code.++ let binary = binary_path();+ if !binary.exists() { return; }++ // Start WITHOUT --no-overlay so the AppKit render loop runs.+ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ // Initialize.+ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Call move_cursor several times to drive the overlay render loop.+ for (id, (x, y)) in [(2, (100.0_f64, 200.0_f64)), (3, (400.0, 300.0)), (4, (800.0, 600.0))] {+ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "id": id,+ "method": "tools/call",+ "params": { "name": "move_cursor", "arguments": { "x": x, "y": y } }+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], id);+ // Should succeed (isError absent or false).+ let is_error = resp["result"]["isError"].as_bool().unwrap_or(false);+ assert!(!is_error, "move_cursor failed: {:?}", resp);+ }++ // Also call get_agent_cursor_state — should report the last position.+ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0",+ "id": 5,+ "method": "tools/call",+ "params": { "name": "get_agent_cursor_state", "arguments": {} }+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 5);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));++ // Give the render loop a chance to tick (two frames ≈ 33 ms).+ std::thread::sleep(Duration::from_millis(100));++ // Process must still be alive.+ assert!(+ child.try_wait().expect("try_wait").is_none(),+ "cua-driver crashed during overlay move_cursor test"+ );++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_all_expected_tools_registered() {+ //! Verify that all tools from the reference implementation are registered.+ //! Adding a new tool to the macOS reference requires adding it to this list.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list"}));+ let resp = read_response(&mut stdout);+ let tools = resp["result"]["tools"].as_array().expect("tools array");+ let names: std::collections::HashSet<&str> = tools.iter()+ .filter_map(|t| t["name"].as_str())+ .collect();++ let expected = [+ "list_apps", "list_windows", "get_window_state", "launch_app",+ "click", "double_click", "right_click", "type_text", "type_text_chars",+ "press_key", "hotkey", "set_value", "scroll", "screenshot", "zoom",+ "get_screen_size", "get_cursor_position",+ "move_cursor", "set_agent_cursor_enabled", "set_agent_cursor_motion",+ "get_agent_cursor_state", "check_permissions", "get_config", "set_config",+ "get_accessibility_tree",+ "set_recording", "get_recording_state", "replay_trajectory",+ "browser_eval",+ ];+ for name in &expected {+ assert!(names.contains(name), "Missing tool: {name} (registered: {names:?})");+ }++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_get_screen_size_and_cursor_position() {+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // get_screen_size+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"get_screen_size","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "get_screen_size failed: {resp:?}");+ let sc = &resp["result"]["structuredContent"];+ assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0, "width should be positive");+ assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0, "height should be positive");++ // get_cursor_position+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"get_cursor_position","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "get_cursor_position failed: {resp:?}");+ // x and y may be 0,0 or any value — just verify the keys exist and are numbers.+ assert!(resp["result"]["structuredContent"]["x"].is_number(), "x should be a number");+ assert!(resp["result"]["structuredContent"]["y"].is_number(), "y should be a number");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_get_config_and_check_permissions() {+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"get_config","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));+ assert_eq!(resp["result"]["structuredContent"]["platform"], "macos");++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"check_permissions","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));+ // Returns structured content with accessibility and screen_recording booleans.+ assert!(resp["result"]["structuredContent"]["accessibility"].is_boolean());++ // set_config — change capture_mode to "ax" and verify get_config reflects the new value.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"set_config","arguments":{"capture_mode": "ax", "max_image_dimension": 1920}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from set_config: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "set_config returned empty content");++ // get_config should now reflect the updated values.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"get_config","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["result"]["structuredContent"]["capture_mode"], "ax",+ "get_config should reflect set_config change");+ assert_eq!(resp["result"]["structuredContent"]["max_image_dimension"], 1920,+ "get_config should reflect max_image_dimension change");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_multi_cursor_instance_state() {+ //! Two cursor instances can be created with different IDs; each has independent state.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Move cursor "agent1" to (100, 200).+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"x":100.0,"y":200.0,"cursor_id":"agent1"}}+ }));+ let r2 = read_response(&mut stdout);+ assert!(!r2["result"]["isError"].as_bool().unwrap_or(false));++ // Move cursor "agent2" to (300, 400).+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"x":300.0,"y":400.0,"cursor_id":"agent2"}}+ }));+ let r3 = read_response(&mut stdout);+ assert!(!r3["result"]["isError"].as_bool().unwrap_or(false));++ // Configure cursor "agent1" with a custom color.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"set_agent_cursor_motion","arguments":{+ "cursor_id":"agent1","cursor_color":"#FF0000","cursor_label":"AI-1"+ }}+ }));+ let r4 = read_response(&mut stdout);+ assert!(!r4["result"]["isError"].as_bool().unwrap_or(false));++ // Hide cursor "agent2".+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"set_agent_cursor_enabled","arguments":{"enabled":false,"cursor_id":"agent2"}}+ }));+ let r5 = read_response(&mut stdout);+ assert!(!r5["result"]["isError"].as_bool().unwrap_or(false));++ // get_agent_cursor_state — should have multiple instances.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":6,"method":"tools/call",+ "params":{"name":"get_agent_cursor_state","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));+ let cursors = &resp["result"]["structuredContent"]["cursors"];+ assert!(cursors.as_array().map(|a| a.len() >= 2).unwrap_or(false),+ "Expected at least 2 cursor instances, got: {cursors:?}");++ std::thread::sleep(Duration::from_millis(50));+ assert!(child.try_wait().expect("try_wait").is_none(), "cua-driver crashed during multi-cursor test");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_set_agent_cursor_motion_bezier_knobs() {+ //! set_agent_cursor_motion with Bezier/timing knobs — verifies schema accepts them+ //! and returns a non-error response with the updated values in the response text.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Set Bezier motion knobs including integer-encoded numbers (common from MCP clients).+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_agent_cursor_motion","arguments":{+ "cursor_id": "agent1",+ "arc_size": 0.4,+ "spring": 0.85,+ "glide_duration_ms": 500,+ "dwell_after_click_ms": 200,+ "idle_hide_ms": 5000+ }}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from set_agent_cursor_motion: {resp:?}");+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false),+ "set_agent_cursor_motion returned isError: {resp:?}");+ // Response text should mention the new glide duration.+ let text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(text.contains("500") || text.contains("motion"),+ "Expected motion summary in response, got: {text}");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_concurrent_clients_with_cursor_moves() {+ //! Two concurrent cua-driver processes, each driving their own cursor, should not crash.+ //! This covers the multi-cursor use case where two Codex agents run simultaneously.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut processes: Vec<(std::process::Child, usize)> = (0..2).map(|i| {+ let child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");+ (child, i)+ }).collect();++ // Initialize all processes.+ for (child, i) in &mut processes {+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());+ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 1, "Process {i} initialize failed");+ }++ // Each process moves its cursor to different positions.+ let positions = vec![(100.0_f64, 200.0_f64), (500.0, 600.0)];+ for ((child, i), (px, py)) in processes.iter_mut().zip(positions.iter()) {+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"x":px,"y":py}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false),+ "Process {i} move_cursor failed: {resp:?}");+ }++ std::thread::sleep(Duration::from_millis(100));++ for (child, i) in &mut processes {+ assert!(child.try_wait().expect("try_wait").is_none(),+ "cua-driver process {i} crashed during concurrent cursor test");+ child.kill().ok();+ }+}++#[test]+#[cfg(target_os = "macos")]+fn test_zoom_tool_returns_jpeg() {+ //! Call zoom on a visible window and verify the result contains a JPEG image.+ //! Skips gracefully if no windows are visible (headless environment).+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // List on-screen windows only — off-screen/minimized windows may not be capturable.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only": true}}+ }));+ let resp = read_response(&mut stdout);+ let windows = resp["result"]["structuredContent"]["windows"]+ .as_array()+ .cloned()+ .unwrap_or_default();++ // Try windows until one captures successfully.+ let mut found_jpeg = false;+ let mut tried = 0usize;+ for win in windows.iter().take(5) {+ let Some(wid) = win["window_id"].as_u64() else { continue; };+ tried += 1;++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id": 2 + tried as u64,"method":"tools/call",+ "params":{"name":"zoom","arguments":{+ "window_id": wid,+ "x1": 0, "y1": 0, "x2": 100, "y2": 100+ }}+ }));+ let r = read_response(&mut stdout);+ if r["result"]["isError"].as_bool().unwrap_or(false) {+ // This window might be off-screen or not capturable — try the next.+ continue;+ }+ let content = r["result"]["content"].as_array().expect("content array");+ found_jpeg = content.iter().any(|c| {+ c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"+ && c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)+ });+ if found_jpeg {+ // Also verify structuredContent.+ let sc = &r["result"]["structuredContent"];+ assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive width: {sc:?}");+ assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive height: {sc:?}");+ assert_eq!(sc["format"].as_str().unwrap_or(""), "jpeg", "Expected format=jpeg: {sc:?}");+ break;+ }+ }++ if tried == 0 {+ eprintln!("No on-screen windows found — skipping zoom test");+ child.kill().ok();+ return;+ }+ assert!(found_jpeg, "Expected at least one window to return a valid JPEG from zoom");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_type_text_chars_tool() {+ //! Verify type_text_chars with delay_ms is accepted without error (dry-run via TextEdit or+ //! a pid that accepts WM_CHAR). We just verify the tool responds with a non-error.+ //! Skips gracefully if no visible TextEdit window.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // list_apps to find a target (e.g. TextEdit).+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_apps","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let apps = resp["result"]["structuredContent"]["apps"].as_array();+ let textedit_pid = apps.and_then(|arr| {+ arr.iter().find(|a| a["name"].as_str().unwrap_or("").contains("TextEdit"))+ .and_then(|a| a["pid"].as_i64())+ });++ let Some(pid) = textedit_pid else {+ eprintln!("TextEdit not running — skipping type_text_chars test");+ child.kill().ok();+ return;+ };++ // type_text_chars with delay_ms=5 — just verify the tool invocation is accepted.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"type_text_chars","arguments":{+ "pid": pid, "text": "hi", "delay_ms": 5+ }}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false),+ "type_text_chars returned error: {resp:?}");+ let msg = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(msg.contains("Typed"), "Unexpected message: {msg}");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_recording_session() {+ //! Enable recording, invoke move_cursor (non-read-only), disable, verify action.json written.+ let binary = binary_path();+ if !binary.exists() { return; }++ let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-rec-test-{}", std::process::id()));+ let tmp_str = tmp_dir.to_string_lossy().to_string();++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Enable recording.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{"enabled":true,"output_dir":tmp_str}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "set_recording failed: {resp:?}");+ assert!(resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(false));++ // Invoke move_cursor — should be recorded.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"x":100.0,"y":200.0}}+ }));+ read_response(&mut stdout);++ // get_recording_state — next_turn should be 2.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"get_recording_state","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let next_turn = resp["result"]["structuredContent"]["next_turn"].as_u64().unwrap_or(0);+ assert!(next_turn >= 2, "expected next_turn >= 2, got {next_turn}");++ // Disable recording.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{"enabled":false}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(true));+ child.kill().ok();++ // Small sync wait for the write to flush.+ std::thread::sleep(Duration::from_millis(50));++ // Verify turn-00001/action.json was written.+ let action_path = tmp_dir.join("turn-00001").join("action.json");+ assert!(action_path.exists(), "Expected {action_path:?} to exist");+ let content: serde_json::Value = serde_json::from_str(+ &std::fs::read_to_string(&action_path).unwrap()+ ).unwrap();+ assert_eq!(content["tool"].as_str().unwrap_or(""), "move_cursor");+ assert_eq!(content["arguments"]["x"].as_f64().unwrap_or(0.0), 100.0);++ let _ = std::fs::remove_dir_all(&tmp_dir);+}++#[test]+#[cfg(target_os = "macos")]+fn test_recording_screenshot_capture() {+ //! When recording is active and a tool call includes a window_id, a screenshot.png+ //! should appear alongside action.json in the turn folder.+ let binary = binary_path();+ if !binary.exists() { return; }++ let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-rec-ss-{}", std::process::id()));+ let tmp_str = tmp_dir.to_string_lossy().to_string();++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Find a capturable on-screen window.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only":true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array().cloned().unwrap_or_default();+ // Prefer windows with non-empty titles (more likely capturable).+ let win = wins.iter()+ .find(|w| w["pid"].as_i64().is_some() && !w["title"].as_str().unwrap_or("").is_empty())+ .or_else(|| wins.iter().find(|w| w["pid"].as_i64().is_some()));+ let Some(win) = win else {+ eprintln!("No on-screen windows — skipping recording screenshot test");+ child.kill().ok();+ return;+ };+ let window_id = win["window_id"].as_u64().unwrap();+ let pid = win["pid"].as_i64().unwrap();++ // Enable recording.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{"enabled":true,"output_dir":tmp_str}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "set_recording failed: {resp:?}");++ // Invoke click (non-read-only) with window_id + pid — recording should capture screenshot.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"click","arguments":{"pid": pid, "window_id": window_id, "x":0.0,"y":0.0}}+ }));+ read_response(&mut stdout); // result may be error if coords are off-window — that's ok++ // Disable recording.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{"enabled":false}}+ }));+ read_response(&mut stdout);+ child.kill().ok();++ std::thread::sleep(Duration::from_millis(100));++ // action.json must exist.+ let turn_dir = tmp_dir.join("turn-00001");+ let action_path = turn_dir.join("action.json");+ assert!(action_path.exists(), "Expected action.json at {action_path:?}");+ let content: serde_json::Value = serde_json::from_str(+ &std::fs::read_to_string(&action_path).unwrap()+ ).unwrap();+ assert_eq!(content["tool"].as_str().unwrap_or(""), "click");++ // screenshot.png should exist if screencapture succeeded for this window.+ let ss_path = turn_dir.join("screenshot.png");+ if !ss_path.exists() {+ eprintln!("screenshot.png not written for window {window_id} (may be uncapturable) — skipping PNG check");+ } else {+ let ss_bytes = std::fs::read(&ss_path).unwrap();+ assert!(!ss_bytes.is_empty(), "screenshot.png should not be empty");+ assert_eq!(&ss_bytes[..4], b"\x89PNG", "screenshot.png should be a valid PNG");++ // click.png should also exist (click at 0,0 with crosshair marker).+ let click_path = turn_dir.join("click.png");+ if !click_path.exists() {+ eprintln!("click.png not written — skipping click PNG check");+ } else {+ let click_bytes = std::fs::read(&click_path).unwrap();+ assert!(!click_bytes.is_empty(), "click.png should not be empty");+ assert_eq!(&click_bytes[..4], b"\x89PNG", "click.png should be a valid PNG");+ }+ }++ let _ = std::fs::remove_dir_all(&tmp_dir);+}++#[test]+#[cfg(target_os = "macos")]+fn test_replay_trajectory() {+ //! Write a minimal trajectory (one move_cursor turn), replay it, verify succeeded=1.+ let binary = binary_path();+ if !binary.exists() { return; }++ let traj_dir = std::env::temp_dir().join(format!("cua-driver-rs-replay-{}", std::process::id()));+ let turn_dir = traj_dir.join("turn-00001");+ std::fs::create_dir_all(&turn_dir).unwrap();+ std::fs::write(+ turn_dir.join("action.json"),+ r#"{"tool":"move_cursor","arguments":{"x":50.0,"y":60.0}}"#,+ ).unwrap();++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"replay_trajectory","arguments":{+ "dir": traj_dir.to_string_lossy(),+ "delay_ms": 0+ }}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "replay error: {resp:?}");+ let sc = &resp["result"]["structuredContent"];+ assert_eq!(sc["attempted"].as_u64().unwrap_or(0), 1, "expected attempted=1");+ assert_eq!(sc["succeeded"].as_u64().unwrap_or(0), 1, "expected succeeded=1");+ assert_eq!(sc["failed"].as_u64().unwrap_or(99), 0, "expected failed=0");++ child.kill().ok();+ let _ = std::fs::remove_dir_all(&traj_dir);+}++#[test]+#[cfg(target_os = "macos")]+fn test_browser_eval_no_cdp_error() {+ //! Verify browser_eval is registered and returns a meaningful error when+ //! no Chromium process is listening on the given cdp_port.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Use a port that is almost certainly not listening.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"browser_eval","arguments":{+ "expression": "1+1",+ "cdp_port": 19722+ }}+ }));+ let resp = read_response(&mut stdout);++ // The tool should return isError=true with a connection-refused message.+ assert!(+ resp["result"]["isError"].as_bool().unwrap_or(false),+ "expected isError=true when CDP port is not listening: {resp:?}"+ );+ let text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(+ text.contains("browser_eval"),+ "error text should mention browser_eval: {text}"+ );++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_screenshot_no_window_id() {+ //! Call screenshot without window_id — should capture the full display and return a PNG.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"screenshot","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from screenshot: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "screenshot should return at least one content item");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_screenshot_jpeg_format() {+ //! Call screenshot with format=jpeg — should return a JPEG image.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"screenshot","arguments":{"format":"jpeg","quality":70}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from screenshot: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ let is_error = resp["result"]["isError"].as_bool().unwrap_or(false);+ if !is_error {+ let has_jpeg = content.iter().any(|c| {+ c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"+ && c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)+ });+ assert!(has_jpeg, "Expected image/jpeg in screenshot response, got: {content:?}");+ // Verify structured content has width and height.+ let sc = &resp["result"]["structuredContent"];+ assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive width in structuredContent");+ assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive height in structuredContent");+ assert_eq!(sc["format"].as_str().unwrap_or(""), "jpeg", "Expected format=jpeg");+ }++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_hotkey_keys_array() {+ //! Verify hotkey accepts a keys array (["cmd","a"]) without a protocol error.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let windows = resp["result"]["structuredContent"]["windows"].as_array();+ let first_win = windows.and_then(|a| a.iter().find(|w| w["pid"].as_i64().is_some()));++ let Some(win) = first_win else {+ eprintln!("No windows found — skipping hotkey keys array test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ // keys array should be parsed and the tool invoked; result may succeed or fail depending+ // on whether the target app accepts the hotkey, but there must be no protocol error.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"hotkey","arguments":{+ "pid": pid, "window_id": wid, "keys": ["cmd","a"]+ }}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from hotkey: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "hotkey should return content");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_get_accessibility_tree() {+ //! get_accessibility_tree returns a lightweight process+window snapshot.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"get_accessibility_tree","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error: {resp:?}");+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false),+ "get_accessibility_tree returned error: {resp:?}");++ let sc = &resp["result"]["structuredContent"];+ // macOS backend returns "apps"; Linux returns "processes". Either key is valid.+ let has_apps = sc["apps"].is_array() || sc["processes"].is_array();+ assert!(has_apps, "Expected apps or processes array in structured content, got: {sc:?}");++ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "Expected non-empty content");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_get_window_state_ax_mode() {+ //! get_window_state with capture_mode="ax" should return a tree but no screenshot image.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let windows = resp["result"]["structuredContent"]["windows"].as_array();+ let first_win = windows.and_then(|a| a.iter().find(|w| {+ w["pid"].as_i64().is_some() && w["window_id"].as_u64().is_some()+ }));++ let Some(win) = first_win else {+ eprintln!("No windows found — skipping get_window_state ax test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"get_window_state","arguments":{+ "pid": pid, "window_id": wid, "capture_mode": "ax"+ }}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from get_window_state: {resp:?}");++ let content = resp["result"]["content"].as_array().expect("content array");+ // ax mode must NOT return an image content item.+ let has_image = content.iter().any(|c| c["type"] == "image");+ assert!(!has_image, "capture_mode=ax should not return an image, got: {content:?}");+ assert!(!content.is_empty(), "Expected at least one content item");++ // structuredContent must include window_id, pid, element_count.+ let sc = &resp["result"]["structuredContent"];+ assert_eq!(sc["window_id"].as_u64().unwrap_or(0), wid, "structuredContent.window_id mismatch");+ assert_eq!(sc["pid"].as_i64().unwrap_or(0), pid, "structuredContent.pid mismatch");+ assert!(sc["element_count"].is_number(), "expected element_count in structuredContent: {sc:?}");+ // ax mode — screenshot_width should NOT be present.+ assert!(sc["screenshot_width"].is_null(), "ax mode should not have screenshot_width: {sc:?}");++ // Also test vision mode: screenshot_width/height should be present.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"get_window_state","arguments":{+ "pid": pid, "window_id": wid, "capture_mode": "vision"+ }}+ }));+ let vision_resp = read_response(&mut stdout);+ assert!(vision_resp["error"].is_null(), "Protocol error from vision get_window_state: {vision_resp:?}");+ if !vision_resp["result"]["isError"].as_bool().unwrap_or(false) {+ let vsc = &vision_resp["result"]["structuredContent"];+ assert!(vsc["screenshot_width"].as_f64().unwrap_or(0.0) > 0.0,+ "vision mode should have screenshot_width: {vsc:?}");+ assert!(vsc["screenshot_height"].as_f64().unwrap_or(0.0) > 0.0,+ "vision mode should have screenshot_height: {vsc:?}");+ }++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_type_text_chars_type_chars_only_schema() {+ //! Verify type_text_chars schema includes type_chars_only param (cross-platform parity),+ //! and that passing type_chars_only=true doesn't cause an additionalProperties error.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Verify schema contains type_chars_only via tools/list.+ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}));+ let list_resp = read_response(&mut stdout);+ let tools = list_resp["result"]["tools"].as_array().expect("tools array");+ let ttc = tools.iter().find(|t| t["name"] == "type_text_chars")+ .expect("type_text_chars not found in tools/list");+ let props = &ttc["inputSchema"]["properties"];+ assert!(props["type_chars_only"].is_object(),+ "type_text_chars schema missing type_chars_only property: {props:?}");++ // Also verify list_windows schema has on_screen_only.+ let lw = tools.iter().find(|t| t["name"] == "list_windows")+ .expect("list_windows not found in tools/list");+ let lw_props = &lw["inputSchema"]["properties"];+ assert!(lw_props["on_screen_only"].is_object(),+ "list_windows schema missing on_screen_only property: {lw_props:?}");++ // Verify set_agent_cursor_motion has Bezier motion knobs.+ let sacm = tools.iter().find(|t| t["name"] == "set_agent_cursor_motion")+ .expect("set_agent_cursor_motion not found in tools/list");+ let sacm_props = &sacm["inputSchema"]["properties"];+ for knob in &["arc_size", "spring", "glide_duration_ms", "dwell_after_click_ms", "idle_hide_ms"] {+ assert!(sacm_props[knob].is_object(),+ "set_agent_cursor_motion schema missing {knob}: {sacm_props:?}");+ }+ // Also verify set_config schema has the enum restriction.+ let sc = tools.iter().find(|t| t["name"] == "set_config")+ .expect("set_config not found in tools/list");+ let sc_props = &sc["inputSchema"]["properties"];+ assert!(sc_props["capture_mode"]["enum"].is_array(),+ "set_config capture_mode should have enum: {sc_props:?}");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_list_windows_structured_content() {+ //! Verify list_windows returns structuredContent.windows array with expected fields.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error: {resp:?}");+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false),+ "list_windows returned error: {resp:?}");++ let sc = &resp["result"]["structuredContent"];+ assert!(sc["windows"].is_array(), "Expected windows array in structuredContent: {sc:?}");++ // If there are any windows, verify the expected fields are present.+ if let Some(wins) = sc["windows"].as_array() {+ if let Some(w) = wins.first() {+ assert!(w["window_id"].is_number(), "window_id missing from window: {w:?}");+ assert!(w["pid"].is_number(), "pid missing from window: {w:?}");+ }+ }++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_press_key_harmless() {+ //! press_key with a harmless key (F24 — virtually no app responds to it) sent to the+ //! first available window. Just verifies the tool doesn't return a protocol error.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Find a window to target.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.iter().find(|w| w["pid"].as_i64().is_some())) else {+ eprintln!("No windows found — skipping press_key test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ // F24 is a function key that virtually no app binds — safe to send.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"press_key","arguments":{"pid": pid, "window_id": wid, "key": "F24"}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from press_key: {resp:?}");+ // Tool may succeed or return an app error, but must NOT be a protocol-level error.+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "press_key returned empty content");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_scroll_tool() {+ //! scroll with direction=down, by=line, amount=1 against the first available window.+ //! Verifies the tool is accepted and returns content, not a protocol error.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Find a pid with a window.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_apps","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let apps = resp["result"]["structuredContent"]["apps"].as_array();+ let Some(app) = apps.and_then(|a| a.iter().find(|a| a["pid"].as_i64().is_some())) else {+ eprintln!("No apps found — skipping scroll test");+ child.kill().ok();+ return;+ };+ let pid = app["pid"].as_i64().unwrap();++ // scroll with minimal parameters — window_id is auto-resolved from pid.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"scroll","arguments":{"pid": pid, "direction": "down", "amount": 1}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from scroll: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "scroll returned empty content");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_click_pixel_path() {+ //! click at window-local (5, 5) — title bar area, safe to click without disrupting UI.+ //! Verifies the pixel-coordinate path for click works without a protocol error.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Find an on-screen window.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only": true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.iter().find(|w| w["pid"].as_i64().is_some())) else {+ eprintln!("No on-screen windows — skipping click pixel path test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ // Click at (5, 5) in window-local coords — title bar, harmless.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"click","arguments":{"pid": pid, "window_id": wid, "x": 5.0, "y": 5.0}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from click: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "click returned empty content");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_type_text_tool() {+ //! Opens TextEdit (or reuses it if already running) and types a short string via type_text.+ //! Skips gracefully if TextEdit is not available or has no window.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Launch TextEdit (no-op if already running).+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"launch_app","arguments":{"name":"TextEdit"}}+ }));+ read_response(&mut stdout);+ std::thread::sleep(std::time::Duration::from_millis(800));++ // Find TextEdit's pid.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"list_apps","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let apps = resp["result"]["structuredContent"]["apps"].as_array();+ let Some(app) = apps.and_then(|a| {+ a.iter().find(|a| {+ a["name"].as_str().map(|n| n.eq_ignore_ascii_case("textedit")).unwrap_or(false)+ })+ }) else {+ eprintln!("TextEdit not found — skipping type_text test");+ child.kill().ok();+ return;+ };+ let pid = app["pid"].as_i64().unwrap();++ // Find TextEdit's first on-screen window.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"pid": pid, "on_screen_only": true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.first()) else {+ eprintln!("TextEdit has no on-screen windows — skipping type_text test");+ child.kill().ok();+ return;+ };+ let wid = win["window_id"].as_u64().unwrap();++ // type_text into TextEdit's window.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"type_text","arguments":{"pid": pid, "window_id": wid, "text": "cua-test"}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from type_text: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "type_text returned empty content");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_double_click_and_right_click_pixel_path() {+ //! double_click and right_click at title-bar coords — verifies both tools accept pixel path.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only": true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.iter().find(|w| w["pid"].as_i64().is_some())) else {+ eprintln!("No on-screen windows — skipping double/right click test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ // double_click at safe coords.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"double_click","arguments":{"pid": pid, "window_id": wid, "x": 5.0, "y": 5.0}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from double_click: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true),+ "double_click returned empty content");++ // right_click at same coords.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"right_click","arguments":{"pid": pid, "window_id": wid, "x": 5.0, "y": 5.0}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from right_click: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true),+ "right_click returned empty content");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_set_value_via_element_index() {+ //! get_window_state on TextEdit → find a text-area element → set_value on it.+ //! Skips gracefully if TextEdit is not available.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Ensure TextEdit is running.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"launch_app","arguments":{"name":"TextEdit"}}+ }));+ read_response(&mut stdout);+ std::thread::sleep(std::time::Duration::from_millis(800));++ // Find TextEdit.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"list_apps","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let apps = resp["result"]["structuredContent"]["apps"].as_array();+ let Some(app) = apps.and_then(|a| {+ a.iter().find(|a| a["name"].as_str()+ .map(|n| n.eq_ignore_ascii_case("textedit")).unwrap_or(false))+ }) else {+ eprintln!("TextEdit not found — skipping set_value test");+ child.kill().ok();+ return;+ };+ let pid = app["pid"].as_i64().unwrap();++ // Get first window.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"pid": pid, "on_screen_only": true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.first()) else {+ eprintln!("TextEdit has no on-screen windows — skipping set_value test");+ child.kill().ok();+ return;+ };+ let wid = win["window_id"].as_u64().unwrap();++ // Walk AX tree to get element_count — we'll target element_index 0.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"get_window_state","arguments":{"pid": pid, "window_id": wid, "capture_mode": "ax"}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from get_window_state: {resp:?}");+ let element_count = resp["result"]["structuredContent"]["element_count"].as_u64().unwrap_or(0);+ if element_count == 0 {+ eprintln!("No AX elements — skipping set_value test");+ child.kill().ok();+ return;+ }++ // set_value on element 0 — this is the document / text area in a new TextEdit document.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":6,"method":"tools/call",+ "params":{"name":"set_value","arguments":{"pid": pid, "window_id": wid, "element_index": 0, "value": "cua-test-value"}}+ }));+ let resp = read_response(&mut stdout);+ // set_value may fail if element 0 is not settable; accept tool-level errors but not protocol errors.+ assert!(resp["error"].is_null(), "Protocol error from set_value: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "set_value returned empty content");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_click_debug_image_out() {+ //! click with debug_image_out writes a PNG crosshair file and then proceeds.+ //! Verifies the debug capture path works end-to-end.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only": true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ // Prefer a window with a non-empty title to avoid uncapturable overlay/system windows.+ let Some(win) = wins.and_then(|a| a.iter().find(|w| {+ w["pid"].as_i64().is_some() && !w["title"].as_str().unwrap_or("").is_empty()+ }).or_else(|| a.iter().find(|w| w["pid"].as_i64().is_some()))) else {+ eprintln!("No on-screen windows — skipping debug_image_out test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ let dbg_path = std::env::temp_dir().join(format!("cua-driver-rs-dbg-{}.png", std::process::id()));+ let dbg_path_str = dbg_path.to_string_lossy().to_string();++ // Click with debug_image_out — should write the PNG and then proceed.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"click","arguments":{+ "pid": pid, "window_id": wid,+ "x": 10.0, "y": 10.0,+ "debug_image_out": dbg_path_str+ }}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from click+debug_image_out: {resp:?}");++ // The tool may return isError if screencapture can't capture this particular window+ // (e.g., overlay windows, system windows). If that happens, skip the PNG check.+ let is_error = resp["result"]["isError"].as_bool().unwrap_or(false);+ if is_error {+ let msg = resp["result"]["content"][0]["text"].as_str().unwrap_or("?");+ eprintln!("debug_image_out screenshot failed for this window ({msg}) — skipping PNG check");+ child.kill().ok();+ return;+ }++ // Verify the PNG was actually written.+ assert!(dbg_path.exists(), "debug_image_out PNG was not written to {dbg_path:?}");+ let meta = std::fs::metadata(&dbg_path).expect("metadata");+ assert!(meta.len() > 100, "debug_image_out PNG is suspiciously small");++ let _ = std::fs::remove_file(&dbg_path);+ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_concurrent_multi_driver_isolation() {+ //! Two cua-driver processes running simultaneously with different cursor IDs.+ //! Verifies that concurrent sessions don't interfere — each tracks its own+ //! cursor state and the overlay stays alive under concurrent load.+ let binary = binary_path();+ if !binary.exists() { return; }++ // Spawn two drivers concurrently.+ let mut child_a = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn driver A");+ let mut child_b = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn driver B");++ let stdin_a = child_a.stdin.as_mut().unwrap();+ let mut stdout_a = BufReader::new(child_a.stdout.as_mut().unwrap());+ let stdin_b = child_b.stdin.as_mut().unwrap();+ let mut stdout_b = BufReader::new(child_b.stdout.as_mut().unwrap());++ // Initialize both.+ send_request(stdin_a, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ send_request(stdin_b, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout_a);+ read_response(&mut stdout_b);++ // Driver A: set_agent_cursor_enabled for cursor "alpha", Driver B for cursor "beta".+ send_request(stdin_a, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_agent_cursor_enabled","arguments":{"cursor_id":"alpha","enabled":true}}+ }));+ send_request(stdin_b, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_agent_cursor_enabled","arguments":{"cursor_id":"beta","enabled":false}}+ }));+ let resp_a = read_response(&mut stdout_a);+ let resp_b = read_response(&mut stdout_b);+ assert!(resp_a["error"].is_null(), "Driver A set_agent_cursor_enabled failed: {resp_a:?}");+ assert!(resp_b["error"].is_null(), "Driver B set_agent_cursor_enabled failed: {resp_b:?}");++ // Each driver queries its own cursor state — should reflect what it set.+ // get_agent_cursor_state takes no arguments; returns { "cursors": [...] }.+ send_request(stdin_a, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"get_agent_cursor_state","arguments":{}}+ }));+ send_request(stdin_b, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"get_agent_cursor_state","arguments":{}}+ }));+ let state_a = read_response(&mut stdout_a);+ let state_b = read_response(&mut stdout_b);+ assert!(state_a["error"].is_null(), "Driver A get_agent_cursor_state failed: {state_a:?}");+ assert!(state_b["error"].is_null(), "Driver B get_agent_cursor_state failed: {state_b:?}");++ // cursors is an array of { config: { cursor_id, enabled, ... }, x, y }.+ // Cursor alpha should be enabled (we set true), beta disabled (we set false).+ // The state lives per process — processes don't share CursorRegistry.+ let cursors_a = state_a["result"]["structuredContent"]["cursors"].as_array();+ let cursors_b = state_b["result"]["structuredContent"]["cursors"].as_array();++ let alpha_enabled = cursors_a.and_then(|arr| arr.iter().find(|c| {+ c["config"]["cursor_id"].as_str() == Some("alpha")+ })).and_then(|c| c["config"]["enabled"].as_bool());+ let beta_enabled = cursors_b.and_then(|arr| arr.iter().find(|c| {+ c["config"]["cursor_id"].as_str() == Some("beta")+ })).and_then(|c| c["config"]["enabled"].as_bool());++ assert_eq!(alpha_enabled, Some(true), "Cursor alpha should be enabled in driver A");+ assert_eq!(beta_enabled, Some(false), "Cursor beta should be disabled in driver B");++ // Move cursor on driver A without crashing driver B.+ send_request(stdin_a, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"cursor_id":"alpha","x":100,"y":100}}+ }));+ send_request(stdin_b, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"cursor_id":"beta","x":200,"y":200}}+ }));+ let mv_a = read_response(&mut stdout_a);+ let mv_b = read_response(&mut stdout_b);+ assert!(mv_a["error"].is_null(), "Driver A move_cursor failed: {mv_a:?}");+ assert!(mv_b["error"].is_null(), "Driver B move_cursor failed: {mv_b:?}");++ child_a.kill().ok();+ child_b.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_set_config_screenshot_resize() {+ //! set_config(max_image_dimension=200) then screenshot — returned image must+ //! have both dimensions ≤ 200. Verifies the ResizeRegistry pipeline end-to-end.+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Set a tiny max_image_dimension so the full display screenshot will be resized.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_config","arguments":{"max_image_dimension":200}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "set_config failed: {resp:?}");+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "set_config returned error: {resp:?}");++ // Capture full display; must be ≤ 200 px in both dimensions.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"screenshot","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "screenshot failed: {resp:?}");++ let w = resp["result"]["structuredContent"]["width"].as_u64().unwrap_or(9999);+ let h = resp["result"]["structuredContent"]["height"].as_u64().unwrap_or(9999);+ assert!(w <= 200, "screenshot width {w} should be ≤ 200 after max_image_dimension=200");+ assert!(h <= 200, "screenshot height {h} should be ≤ 200 after max_image_dimension=200");++ // get_config should reflect the updated value.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"get_config","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let dim = resp["result"]["structuredContent"]["max_image_dimension"].as_u64();+ assert_eq!(dim, Some(200), "get_config should reflect max_image_dimension=200");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_zoom_from_zoom_click_round_trip() {+ //! Verify the zoom → from_zoom click pipeline:+ //! 1. click(from_zoom=true) with no zoom context returns the expected error.+ //! 2. zoom() stores context; subsequent click(from_zoom=true) does NOT return+ //! that error (translation succeeded, even if the click itself may fail).+ let binary = binary_path();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // List on-screen windows to find a target window and pid.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only":true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array().cloned().unwrap_or_default();+ let Some(win) = wins.iter().find(|w| w["pid"].as_i64().is_some()) else {+ eprintln!("No on-screen windows — skipping from_zoom test");+ child.kill().ok();+ return;+ };+ let window_id = win["window_id"].as_u64().unwrap();+ let pid = win["pid"].as_i64().unwrap();++ // Step 1: click with from_zoom=true and NO prior zoom context — expect error.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"click","arguments":{"pid": pid, "window_id": window_id, "x":10.0,"y":10.0,"from_zoom":true}}+ }));+ let resp = read_response(&mut stdout);+ let is_err = resp["result"]["isError"].as_bool().unwrap_or(false);+ let err_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(is_err, "Expected error when from_zoom=true with no context, got: {resp:?}");+ assert!(err_text.contains("no zoom context"), "Expected 'no zoom context' error, got: {err_text}");++ // Step 2: call zoom on that window to store context.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"zoom","arguments":{"window_id": window_id, "pid": pid, "x1":0,"y1":0,"x2":50,"y2":50}}+ }));+ let resp = read_response(&mut stdout);+ if resp["result"]["isError"].as_bool().unwrap_or(false) {+ // Window may not be capturable (e.g. system UI) — skip.+ eprintln!("zoom failed — skipping from_zoom click test");+ child.kill().ok();+ return;+ }+ // Verify zoom returned structured content with width/height.+ let sc = &resp["result"]["structuredContent"];+ assert!(sc["width"].as_u64().unwrap_or(0) > 0, "zoom should return positive width: {sc:?}");++ // Step 3: click with from_zoom=true — should NOT return the "no zoom context" error.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"click","arguments":{"pid": pid, "window_id": window_id, "x":10.0,"y":10.0,"from_zoom":true}}+ }));+ let resp = read_response(&mut stdout);+ let is_err = resp["result"]["isError"].as_bool().unwrap_or(false);+ let err_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ // Translation should succeed — if click fails it's for another reason (target app state), not missing context.+ assert!(!err_text.contains("no zoom context"),+ "After zoom(), from_zoom click should not say 'no zoom context', got: {err_text}");+ // (is_err may be true if the click target was invalid; that's acceptable.)+ let _ = is_err;++ child.kill().ok();+}++#[test]+#[cfg(target_os = "macos")]+fn test_set_recording_video_experimental_accepted() {+ //! video_experimental is a valid schema parameter — must be accepted without error.+ let binary = binary_path();+ if !binary.exists() { return; }++ let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-videxp-{}", std::process::id()));+ let tmp_str = tmp_dir.to_string_lossy().to_string();+ std::fs::create_dir_all(&tmp_dir).unwrap();++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Enable recording with video_experimental=true — must not return a schema error.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{+ "enabled": true, "output_dir": tmp_str, "video_experimental": true+ }}+ }));+ let resp = read_response(&mut stdout);+ // Must not be a JSON-RPC protocol error.+ assert!(resp["error"].is_null(), "Expected no JSON-RPC error, got: {resp:?}");+ // Tool result may note the feature is not implemented, but must not be a schema validation error.+ let is_err = resp["result"]["isError"].as_bool().unwrap_or(false);+ assert!(!is_err, "set_recording with video_experimental should succeed (note-only): {resp:?}");++ // Disable recording.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{"enabled":false}}+ }));+ read_response(&mut stdout);+ child.kill().ok();+ let _ = std::fs::remove_dir_all(&tmp_dir);+}++// ─────────────────────────────────────────────────────────────────────────────+// Windows integration tests+//+// Mirrors the macOS tests above with Windows-specific adaptations:+// - binary path includes .exe+// - list_apps returns structuredContent.processes (not .apps)+// - get_config returns platform="windows"+// - check_permissions returns { elevated, uia, post_message } (not accessibility)+// - get_accessibility_tree returns processes + windows keys+// - type_text tests target Notepad instead of TextEdit+// ─────────────────────────────────────────────────────────────────────────────++#[cfg(target_os = "windows")]+fn binary_path_windows() -> std::path::PathBuf {+ binary_path().with_extension("exe")+}++#[test]+#[cfg(target_os = "windows")]+fn test_initialize_handshake_windows() {+ let binary = binary_path_windows();+ if !binary.exists() {+ eprintln!("Binary not found at {:?} — run `cargo build` first", binary);+ return;+ }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn cua-driver");++ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({+ "jsonrpc": "2.0", "id": 1, "method": "initialize",+ "params": {+ "protocolVersion": "2024-11-05",+ "clientInfo": { "name": "test-client", "version": "0.0.1" }+ }+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["jsonrpc"], "2.0");+ assert_eq!(resp["id"], 1);+ assert!(resp["result"]["protocolVersion"].is_string());+ assert_eq!(resp["result"]["serverInfo"]["name"], "cua-driver-rs");++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","method":"notifications/initialized"}));++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list"}));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 2);+ let tools = resp["result"]["tools"].as_array().expect("tools array");+ assert!(!tools.is_empty());++ let tool_names: Vec<&str> = tools.iter()+ .filter_map(|t| t["name"].as_str()).collect();+ for expected in &["list_apps", "list_windows", "get_window_state", "click", "type_text", "press_key"] {+ assert!(tool_names.contains(expected), "Missing tool: {}", expected);+ }++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":3,"method":"unknown/method"}));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 3);+ assert_eq!(resp["error"]["code"], -32601);++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_tools_call_list_apps_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_apps","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 2);+ assert!(resp["result"]["content"].is_array());+ let text = resp["result"]["content"][0]["text"].as_str().unwrap();+ assert!(text.contains("Found"), "Expected process list text, got: {}", text);+ // Windows backend returns "processes" key.+ assert!(resp["result"]["structuredContent"]["processes"].is_array(),+ "Expected processes array: {:?}", resp["result"]["structuredContent"]);++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_tools_call_unknown_tool_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"nonexistent_tool","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let is_error = resp["result"]["isError"].as_bool().unwrap_or(false);+ assert!(is_error, "Expected isError=true for unknown tool");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_all_expected_tools_registered_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list"}));+ let resp = read_response(&mut stdout);+ let tools = resp["result"]["tools"].as_array().expect("tools array");+ let names: std::collections::HashSet<&str> = tools.iter()+ .filter_map(|t| t["name"].as_str()).collect();++ let expected = [+ "list_apps", "list_windows", "get_window_state", "launch_app",+ "click", "double_click", "right_click", "type_text", "type_text_chars",+ "press_key", "hotkey", "set_value", "scroll", "screenshot", "zoom",+ "get_screen_size", "get_cursor_position",+ "move_cursor", "set_agent_cursor_enabled", "set_agent_cursor_motion",+ "get_agent_cursor_state", "check_permissions", "get_config", "set_config",+ "get_accessibility_tree",+ "set_recording", "get_recording_state", "replay_trajectory",+ "browser_eval",+ ];+ for name in &expected {+ assert!(names.contains(name), "Missing tool: {name} (registered: {names:?})");+ }++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_get_screen_size_and_cursor_position_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"get_screen_size","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "get_screen_size failed: {resp:?}");+ let sc = &resp["result"]["structuredContent"];+ assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0);+ assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"get_cursor_position","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "get_cursor_position failed: {resp:?}");+ assert!(resp["result"]["structuredContent"]["x"].is_number());+ assert!(resp["result"]["structuredContent"]["y"].is_number());++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_get_config_and_check_permissions_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"get_config","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));+ assert_eq!(resp["result"]["structuredContent"]["platform"], "windows");++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"check_permissions","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));+ let sc = &resp["result"]["structuredContent"];+ // Windows returns elevated, uia, post_message booleans.+ assert!(sc["uia"].as_bool().unwrap_or(false), "uia should be true: {sc:?}");+ assert!(sc["post_message"].as_bool().unwrap_or(false), "post_message should be true: {sc:?}");+ assert!(sc["elevated"].is_boolean(), "elevated should be a boolean: {sc:?}");++ // set_config — change capture_mode and verify get_config reflects it.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"set_config","arguments":{"capture_mode":"ax","max_image_dimension":1920}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from set_config: {resp:?}");++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"get_config","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["result"]["structuredContent"]["capture_mode"], "ax");+ assert_eq!(resp["result"]["structuredContent"]["max_image_dimension"], 1920);++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_concurrent_clients_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut processes: Vec<std::process::Child> = (0..2).map(|_| {+ Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn")+ }).collect();++ for (i, child) in processes.iter_mut().enumerate() {+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id": i + 1,"method":"initialize","params":{}+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], (i + 1) as i64);+ assert!(resp["result"]["protocolVersion"].is_string(), "Process {i} failed to initialize");+ }++ for mut child in processes { child.kill().ok(); }+}++#[test]+#[cfg(target_os = "windows")]+fn test_list_windows_structured_content_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error: {resp:?}");+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));+ let sc = &resp["result"]["structuredContent"];+ assert!(sc["windows"].is_array(), "Expected windows array: {sc:?}");+ if let Some(wins) = sc["windows"].as_array() {+ if let Some(w) = wins.first() {+ assert!(w["window_id"].is_number(), "window_id missing: {w:?}");+ assert!(w["pid"].is_number(), "pid missing: {w:?}");+ }+ }++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_get_accessibility_tree_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"get_accessibility_tree","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error: {resp:?}");+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));+ let sc = &resp["result"]["structuredContent"];+ // Windows backend returns both "processes" and "windows" keys.+ assert!(sc["processes"].is_array(), "Expected processes array: {sc:?}");+ assert!(sc["windows"].is_array(), "Expected windows array: {sc:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty());++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_screenshot_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Full-display screenshot (no window_id).+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"screenshot","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ assert!(!content.is_empty(), "screenshot returned empty content");++ // JPEG format.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"screenshot","arguments":{"format":"jpeg","quality":70}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from screenshot jpeg: {resp:?}");+ if !resp["result"]["isError"].as_bool().unwrap_or(false) {+ let content = resp["result"]["content"].as_array().expect("content array");+ let has_jpeg = content.iter().any(|c| {+ c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"+ && c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)+ });+ assert!(has_jpeg, "Expected image/jpeg in screenshot response: {content:?}");+ let sc = &resp["result"]["structuredContent"];+ assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0);+ assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0);+ assert_eq!(sc["format"].as_str().unwrap_or(""), "jpeg");+ }++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_zoom_tool_returns_jpeg_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only":true}}+ }));+ let resp = read_response(&mut stdout);+ let windows = resp["result"]["structuredContent"]["windows"]+ .as_array().cloned().unwrap_or_default();++ let mut found_jpeg = false;+ let mut tried = 0usize;+ for win in windows.iter().take(5) {+ let Some(wid) = win["window_id"].as_u64() else { continue; };+ tried += 1;+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id": 2 + tried as u64,"method":"tools/call",+ "params":{"name":"zoom","arguments":{+ "window_id":wid,"x1":0,"y1":0,"x2":100,"y2":100+ }}+ }));+ let r = read_response(&mut stdout);+ if r["result"]["isError"].as_bool().unwrap_or(false) { continue; }+ let content = r["result"]["content"].as_array().expect("content array");+ found_jpeg = content.iter().any(|c| {+ c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"+ && c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)+ });+ if found_jpeg {+ let sc = &r["result"]["structuredContent"];+ assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0);+ assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0);+ assert_eq!(sc["format"].as_str().unwrap_or(""), "jpeg");+ break;+ }+ }++ if tried == 0 {+ eprintln!("No on-screen windows found — skipping zoom test");+ child.kill().ok();+ return;+ }+ assert!(found_jpeg, "Expected at least one window to return a valid JPEG from zoom");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_overlay_move_cursor_stays_alive_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ for (id, (x, y)) in [(2u64, (100.0_f64, 200.0_f64)), (3, (400.0, 300.0)), (4, (800.0, 600.0))] {+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":id,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"x":x,"y":y}}+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], id);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "move_cursor failed: {:?}", resp);+ }++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"get_agent_cursor_state","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["id"], 5);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));++ std::thread::sleep(Duration::from_millis(100));++ assert!(+ child.try_wait().expect("try_wait").is_none(),+ "cua-driver crashed during overlay move_cursor test"+ );++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_multi_cursor_instance_state_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"x":100.0,"y":200.0,"cursor_id":"agent1"}}+ }));+ assert!(!read_response(&mut stdout)["result"]["isError"].as_bool().unwrap_or(false));++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"x":300.0,"y":400.0,"cursor_id":"agent2"}}+ }));+ assert!(!read_response(&mut stdout)["result"]["isError"].as_bool().unwrap_or(false));++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"set_agent_cursor_motion","arguments":{+ "cursor_id":"agent1","cursor_color":"#FF0000","cursor_label":"AI-1"+ }}+ }));+ assert!(!read_response(&mut stdout)["result"]["isError"].as_bool().unwrap_or(false));++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"set_agent_cursor_enabled","arguments":{"enabled":false,"cursor_id":"agent2"}}+ }));+ assert!(!read_response(&mut stdout)["result"]["isError"].as_bool().unwrap_or(false));++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":6,"method":"tools/call",+ "params":{"name":"get_agent_cursor_state","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));+ let cursors = &resp["result"]["structuredContent"]["cursors"];+ assert!(cursors.as_array().map(|a| a.len() >= 2).unwrap_or(false),+ "Expected at least 2 cursor instances: {cursors:?}");++ std::thread::sleep(Duration::from_millis(50));+ assert!(child.try_wait().expect("try_wait").is_none(), "cua-driver crashed during multi-cursor test");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_press_key_harmless_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.iter().find(|w| w["pid"].as_i64().is_some())) else {+ eprintln!("No windows found — skipping press_key test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"press_key","arguments":{"pid":pid,"window_id":wid,"key":"F24"}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from press_key: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true));++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_scroll_tool_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Windows list_apps returns "processes".+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_apps","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let procs = resp["result"]["structuredContent"]["processes"].as_array();+ let Some(proc_) = procs.and_then(|a| a.iter().find(|p| p["pid"].as_i64().is_some())) else {+ eprintln!("No processes found — skipping scroll test");+ child.kill().ok();+ return;+ };+ let pid = proc_["pid"].as_i64().unwrap();++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"scroll","arguments":{"pid":pid,"direction":"down","amount":1}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from scroll: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true));++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_click_pixel_path_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only":true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.iter().find(|w| w["pid"].as_i64().is_some())) else {+ eprintln!("No on-screen windows — skipping click pixel path test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"click","arguments":{"pid":pid,"window_id":wid,"x":5.0,"y":5.0}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from click: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true));++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_double_click_and_right_click_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only":true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.iter().find(|w| w["pid"].as_i64().is_some())) else {+ eprintln!("No on-screen windows — skipping double/right click test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"double_click","arguments":{"pid":pid,"window_id":wid,"x":5.0,"y":5.0}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from double_click: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true));++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"right_click","arguments":{"pid":pid,"window_id":wid,"x":5.0,"y":5.0}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from right_click: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true));++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_hotkey_keys_array_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let windows = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = windows.and_then(|a| a.iter().find(|w| w["pid"].as_i64().is_some())) else {+ eprintln!("No windows found — skipping hotkey test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"hotkey","arguments":{"pid":pid,"window_id":wid,"keys":["ctrl","a"]}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from hotkey: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true));++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_get_window_state_ax_mode_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let windows = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = windows.and_then(|a| a.iter().find(|w| {+ w["pid"].as_i64().is_some() && w["window_id"].as_u64().is_some()+ })) else {+ eprintln!("No windows found — skipping get_window_state ax test");+ child.kill().ok();+ return;+ };+ let pid = win["pid"].as_i64().unwrap();+ let wid = win["window_id"].as_u64().unwrap();++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"get_window_state","arguments":{"pid":pid,"window_id":wid,"capture_mode":"ax"}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error: {resp:?}");+ let content = resp["result"]["content"].as_array().expect("content array");+ let has_image = content.iter().any(|c| c["type"] == "image");+ assert!(!has_image, "capture_mode=ax should not return an image");+ assert!(!content.is_empty());++ let sc = &resp["result"]["structuredContent"];+ assert_eq!(sc["window_id"].as_u64().unwrap_or(0), wid);+ assert_eq!(sc["pid"].as_i64().unwrap_or(0), pid);+ assert!(sc["element_count"].is_number(), "expected element_count: {sc:?}");++ // vision mode should include screenshot dimensions.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"get_window_state","arguments":{"pid":pid,"window_id":wid,"capture_mode":"vision"}}+ }));+ let vision_resp = read_response(&mut stdout);+ assert!(vision_resp["error"].is_null(), "Protocol error from vision get_window_state: {vision_resp:?}");+ if !vision_resp["result"]["isError"].as_bool().unwrap_or(false) {+ let vsc = &vision_resp["result"]["structuredContent"];+ assert!(vsc["screenshot_width"].as_f64().unwrap_or(0.0) > 0.0, "vision mode should have screenshot_width: {vsc:?}");+ }++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_type_text_chars_schema_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}));+ let list_resp = read_response(&mut stdout);+ let tools = list_resp["result"]["tools"].as_array().expect("tools array");++ let ttc = tools.iter().find(|t| t["name"] == "type_text_chars")+ .expect("type_text_chars not found");+ let props = &ttc["inputSchema"]["properties"];+ assert!(props["type_chars_only"].is_object(),+ "type_text_chars schema missing type_chars_only: {props:?}");++ let lw = tools.iter().find(|t| t["name"] == "list_windows").expect("list_windows not found");+ assert!(lw["inputSchema"]["properties"]["on_screen_only"].is_object());++ let sacm = tools.iter().find(|t| t["name"] == "set_agent_cursor_motion")+ .expect("set_agent_cursor_motion not found");+ let sacm_props = &sacm["inputSchema"]["properties"];+ for knob in &["arc_size", "spring", "glide_duration_ms", "dwell_after_click_ms", "idle_hide_ms"] {+ assert!(sacm_props[knob].is_object(), "missing {knob}: {sacm_props:?}");+ }++ let sc = tools.iter().find(|t| t["name"] == "set_config").expect("set_config not found");+ assert!(sc["inputSchema"]["properties"]["capture_mode"]["enum"].is_array());++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_type_text_notepad_windows() {+ //! Launch Notepad, type a short string via type_text. Skips if Notepad unavailable.+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ // Launch Notepad.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"launch_app","arguments":{"name":"notepad.exe"}}+ }));+ read_response(&mut stdout);+ std::thread::sleep(std::time::Duration::from_millis(1000));++ // Find Notepad in running processes.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"list_apps","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let procs = resp["result"]["structuredContent"]["processes"].as_array();+ let Some(proc_) = procs.and_then(|a| {+ a.iter().find(|p| {+ p["name"].as_str().map(|n| n.to_lowercase().contains("notepad")).unwrap_or(false)+ })+ }) else {+ eprintln!("Notepad not found — skipping type_text test");+ child.kill().ok();+ return;+ };+ let pid = proc_["pid"].as_i64().unwrap();++ // Find Notepad's window.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"pid":pid,"on_screen_only":true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array();+ let Some(win) = wins.and_then(|a| a.first()) else {+ eprintln!("Notepad has no on-screen windows — skipping type_text test");+ child.kill().ok();+ return;+ };+ let wid = win["window_id"].as_u64().unwrap();++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"type_text","arguments":{"pid":pid,"window_id":wid,"text":"cua-test"}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Protocol error from type_text: {resp:?}");+ assert!(!resp["result"]["content"].as_array().map(|a| a.is_empty()).unwrap_or(true));++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_recording_session_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-rec-test-{}", std::process::id()));+ let tmp_str = tmp_dir.to_string_lossy().to_string();++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{"enabled":true,"output_dir":tmp_str}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "set_recording failed: {resp:?}");+ assert!(resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(false));++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"move_cursor","arguments":{"x":100.0,"y":200.0}}+ }));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"get_recording_state","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ let next_turn = resp["result"]["structuredContent"]["next_turn"].as_u64().unwrap_or(0);+ assert!(next_turn >= 2, "expected next_turn >= 2, got {next_turn}");++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{"enabled":false}}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(true));+ child.kill().ok();++ std::thread::sleep(Duration::from_millis(50));++ let action_path = tmp_dir.join("turn-00001").join("action.json");+ assert!(action_path.exists(), "Expected action.json at {action_path:?}");+ let content: serde_json::Value = serde_json::from_str(+ &std::fs::read_to_string(&action_path).unwrap()+ ).unwrap();+ assert_eq!(content["tool"].as_str().unwrap_or(""), "move_cursor");+ assert_eq!(content["arguments"]["x"].as_f64().unwrap_or(0.0), 100.0);++ let _ = std::fs::remove_dir_all(&tmp_dir);+}++#[test]+#[cfg(target_os = "windows")]+fn test_replay_trajectory_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let traj_dir = std::env::temp_dir().join(format!("cua-driver-rs-replay-{}", std::process::id()));+ let turn_dir = traj_dir.join("turn-00001");+ std::fs::create_dir_all(&turn_dir).unwrap();+ std::fs::write(+ turn_dir.join("action.json"),+ r#"{"tool":"move_cursor","arguments":{"x":50.0,"y":60.0}}"#,+ ).unwrap();++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"replay_trajectory","arguments":{+ "dir":traj_dir.to_string_lossy(),"delay_ms":0+ }}+ }));+ let resp = read_response(&mut stdout);+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "replay error: {resp:?}");+ let sc = &resp["result"]["structuredContent"];+ assert_eq!(sc["attempted"].as_u64().unwrap_or(0), 1);+ assert_eq!(sc["succeeded"].as_u64().unwrap_or(0), 1);+ assert_eq!(sc["failed"].as_u64().unwrap_or(99), 0);++ child.kill().ok();+ let _ = std::fs::remove_dir_all(&traj_dir);+}++#[test]+#[cfg(target_os = "windows")]+fn test_browser_eval_no_cdp_error_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"browser_eval","arguments":{"expression":"1+1","cdp_port":19722}}+ }));+ let resp = read_response(&mut stdout);+ assert!(+ resp["result"]["isError"].as_bool().unwrap_or(false),+ "expected isError=true when CDP port not listening: {resp:?}"+ );+ let text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(text.contains("browser_eval"), "error text should mention browser_eval: {text}");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_set_config_screenshot_resize_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_config","arguments":{"max_image_dimension":200}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "set_config failed: {resp:?}");+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false));++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"screenshot","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "screenshot failed: {resp:?}");++ let w = resp["result"]["structuredContent"]["width"].as_u64().unwrap_or(9999);+ let h = resp["result"]["structuredContent"]["height"].as_u64().unwrap_or(9999);+ assert!(w <= 200, "screenshot width {w} should be ≤ 200 after max_image_dimension=200");+ assert!(h <= 200, "screenshot height {h} should be ≤ 200 after max_image_dimension=200");++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"get_config","arguments":{}}+ }));+ let resp = read_response(&mut stdout);+ assert_eq!(resp["result"]["structuredContent"]["max_image_dimension"].as_u64(), Some(200));++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_zoom_from_zoom_click_round_trip_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"list_windows","arguments":{"on_screen_only":true}}+ }));+ let resp = read_response(&mut stdout);+ let wins = resp["result"]["structuredContent"]["windows"].as_array().cloned().unwrap_or_default();+ let Some(win) = wins.iter().find(|w| w["pid"].as_i64().is_some()) else {+ eprintln!("No on-screen windows — skipping from_zoom test");+ child.kill().ok();+ return;+ };+ let window_id = win["window_id"].as_u64().unwrap();+ let pid = win["pid"].as_i64().unwrap();++ // click with from_zoom=true and no prior zoom context — expect error.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"click","arguments":{"pid":pid,"window_id":window_id,"x":10.0,"y":10.0,"from_zoom":true}}+ }));+ let resp = read_response(&mut stdout);+ let is_err = resp["result"]["isError"].as_bool().unwrap_or(false);+ let err_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(is_err, "Expected error when from_zoom=true with no context: {resp:?}");+ assert!(err_text.contains("no zoom context"), "Expected 'no zoom context': {err_text}");++ // Call zoom to store context.+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":4,"method":"tools/call",+ "params":{"name":"zoom","arguments":{"window_id":window_id,"pid":pid,"x1":0,"y1":0,"x2":50,"y2":50}}+ }));+ let resp = read_response(&mut stdout);+ if resp["result"]["isError"].as_bool().unwrap_or(false) {+ eprintln!("zoom failed — skipping from_zoom click test");+ child.kill().ok();+ return;+ }++ // click with from_zoom=true should NOT say "no zoom context".+ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":5,"method":"tools/call",+ "params":{"name":"click","arguments":{"pid":pid,"window_id":window_id,"x":10.0,"y":10.0,"from_zoom":true}}+ }));+ let resp = read_response(&mut stdout);+ let err_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(!err_text.contains("no zoom context"),+ "After zoom(), from_zoom click should not say 'no zoom context': {err_text}");++ child.kill().ok();+}++#[test]+#[cfg(target_os = "windows")]+fn test_set_recording_video_experimental_accepted_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-videxp-{}", std::process::id()));+ let tmp_str = tmp_dir.to_string_lossy().to_string();+ std::fs::create_dir_all(&tmp_dir).unwrap();++ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn");+ let stdin = child.stdin.as_mut().unwrap();+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());++ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout);++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{+ "enabled":true,"output_dir":tmp_str,"video_experimental":true+ }}+ }));+ let resp = read_response(&mut stdout);+ assert!(resp["error"].is_null(), "Expected no JSON-RPC error: {resp:?}");+ assert!(!resp["result"]["isError"].as_bool().unwrap_or(false),+ "set_recording with video_experimental should succeed: {resp:?}");++ send_request(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"set_recording","arguments":{"enabled":false}}+ }));+ read_response(&mut stdout);+ child.kill().ok();+ let _ = std::fs::remove_dir_all(&tmp_dir);+}++#[test]+#[cfg(target_os = "windows")]+fn test_concurrent_multi_driver_isolation_windows() {+ let binary = binary_path_windows();+ if !binary.exists() { return; }++ let mut child_a = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn driver A");+ let mut child_b = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn driver B");++ let stdin_a = child_a.stdin.as_mut().unwrap();+ let mut stdout_a = BufReader::new(child_a.stdout.as_mut().unwrap());+ let stdin_b = child_b.stdin.as_mut().unwrap();+ let mut stdout_b = BufReader::new(child_b.stdout.as_mut().unwrap());++ send_request(stdin_a, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ send_request(stdin_b, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ read_response(&mut stdout_a);+ read_response(&mut stdout_b);++ send_request(stdin_a, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_agent_cursor_enabled","arguments":{"cursor_id":"alpha","enabled":true}}+ }));+ send_request(stdin_b, &serde_json::json!({+ "jsonrpc":"2.0","id":2,"method":"tools/call",+ "params":{"name":"set_agent_cursor_enabled","arguments":{"cursor_id":"beta","enabled":false}}+ }));+ let resp_a = read_response(&mut stdout_a);+ let resp_b = read_response(&mut stdout_b);+ assert!(resp_a["error"].is_null(), "Driver A failed: {resp_a:?}");+ assert!(resp_b["error"].is_null(), "Driver B failed: {resp_b:?}");++ send_request(stdin_a, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"get_agent_cursor_state","arguments":{}}+ }));+ send_request(stdin_b, &serde_json::json!({+ "jsonrpc":"2.0","id":3,"method":"tools/call",+ "params":{"name":"get_agent_cursor_state","arguments":{}}+ }));+ let state_a = read_response(&mut stdout_a);+ let state_b = read_response(&mut stdout_b);+ assert!(state_a["error"].is_null());+ assert!(state_b["error"].is_null());++ let cursors_a = state_a["result"]["structuredContent"]["cursors"].as_array();+ let cursors_b = state_b["result"]["structuredContent"]["cursors"].as_array();++ let alpha_enabled = cursors_a.and_then(|arr| arr.iter().find(|c| {+ c["config"]["cursor_id"].as_str() == Some("alpha")+ })).and_then(|c| c["config"]["enabled"].as_bool());+ let beta_enabled = cursors_b.and_then(|arr| arr.iter().find(|c| {+ c["config"]["cursor_id"].as_str() == Some("beta")+ })).and_then(|c| c["config"]["enabled"].as_bool());++ assert_eq!(alpha_enabled, Some(true), "Cursor alpha should be enabled in driver A");+ assert_eq!(beta_enabled, Some(false), "Cursor beta should be disabled in driver B");++ child_a.kill().ok();+ child_b.kill().ok();+}diff --git a/libs/cua-driver-rs/crates/cua-driver/tests/ux_guard_test.rs b/libs/cua-driver-rs/crates/cua-driver/tests/ux_guard_test.rsnew file mode 100644index 0000000000..adeb9d697e--- /dev/null+++ b/libs/cua-driver-rs/crates/cua-driver/tests/ux_guard_test.rs@@ -0,0 +1,545 @@+//! UX-guard integration tests for Windows.+//!+//! These mirror the macOS Python tests in libs/cua-driver/Tests/integration/:+//! - test_background_focus.py+//! - test_click_opens_new_window.py+//! - test_launch_app_visible.py+//! - test_background_menu_shortcut.py+//!+//! Invariant under test (the "UX guard"):+//! The agent must be able to click, type, and launch apps in background+//! windows WITHOUT stealing focus from the user's foreground window.+//!+//! Background target: desktop-test-app-electron (prebuilt binary at+//! test-apps/desktop-test-app-electron.0.1.0.exe). Notepad is used only+//! as a secondary fallback check.+//!+//! All tests:+//! 1. Launch focus-monitor-win (the "user's foreground window").+//! 2. Perform background operations via cua-driver.+//! 3. Assert focus-monitor-win's act_losses counter stayed at 0.+//!+//! Run in sandbox via:+//! .\sandbox\run-tests-in-sandbox.ps1 ux_guard++#![cfg(target_os = "windows")]++use std::io::{BufRead, BufReader, Write};+use std::path::PathBuf;+use std::process::{Child, Command, Stdio};+use std::time::Duration;++// ── helpers shared with mcp_protocol_test ────────────────────────────────────++fn binary_path() -> PathBuf {+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");+ PathBuf::from(manifest)+ .parent().unwrap()+ .parent().unwrap()+ .join("target/debug/cua-driver.exe")+}++fn focus_monitor_path() -> PathBuf {+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");+ PathBuf::from(manifest)+ .parent().unwrap()+ .parent().unwrap()+ .join("target/debug/focus-monitor-win.exe")+}++fn test_app_path() -> PathBuf {+ // In sandbox, sandbox-runner.ps1 copies the exe to %TEMP% to avoid the+ // ShellExecuteW zone-security dialog that blocks on mapped-folder exes.+ if let Ok(p) = std::env::var("TEST_APP_EXE") {+ let pb = PathBuf::from(p);+ if pb.exists() { return pb; }+ }+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");+ PathBuf::from(manifest)+ .parent().unwrap()+ .parent().unwrap()+ .join("test-apps/desktop-test-app-electron.0.1.0.exe")+}++/// Launch the electron test app in the background and return (process, pid).+/// Waits up to 10s for the app's HTTP health endpoint to respond.+/// Returns None if the binary doesn't exist or the app fails to start.+fn launch_test_app() -> Option<(Child, u32)> {+ let exe = test_app_path();+ if !exe.exists() {+ eprintln!("desktop-test-app-electron not found at {exe:?} — skipping");+ return None;+ }+ let child = Command::new(&exe)+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .spawn()+ .ok()?;+ let pid = child.id();+ // Poll the HTTP health endpoint until it responds or timeout.+ // Allow extra time for cold-start in sandbox (no cached Electron DLLs).+ let deadline = std::time::Instant::now() + Duration::from_secs(20);+ loop {+ if std::time::Instant::now() > deadline {+ eprintln!("desktop-test-app-electron HTTP /health did not respond within 10s");+ return Some((child, pid));+ }+ if let Ok(stream) = std::net::TcpStream::connect("127.0.0.1:6769") {+ drop(stream);+ break;+ }+ std::thread::sleep(Duration::from_millis(200));+ }+ // Extra settle time for the window to appear.+ std::thread::sleep(Duration::from_millis(500));+ Some((child, pid))+}++fn loss_file() -> PathBuf {+ std::env::temp_dir().join("focus_monitor_losses.txt")+}+fn key_loss_file() -> PathBuf {+ std::env::temp_dir().join("focus_monitor_key_losses.txt")+}++fn read_losses(path: &std::path::Path) -> u32 {+ std::fs::read_to_string(path)+ .ok()+ .and_then(|s| s.trim().parse().ok())+ .unwrap_or(0)+}++fn send_rpc(stdin: &mut impl Write, req: &serde_json::Value) {+ let line = serde_json::to_string(req).unwrap();+ writeln!(stdin, "{line}").unwrap();+}++fn recv_rpc(reader: &mut impl BufRead) -> serde_json::Value {+ let mut line = String::new();+ reader.read_line(&mut line).expect("read_line");+ serde_json::from_str(line.trim()).expect("parse JSON")+}++/// Spawn cua-driver and do the MCP initialize handshake.+fn spawn_driver() -> (Child, std::process::ChildStdin, BufReader<std::process::ChildStdout>) {+ let binary = binary_path();+ let mut child = Command::new(&binary)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().expect("spawn cua-driver");+ let stdin = child.stdin.take().unwrap();+ let stdout = BufReader::new(child.stdout.take().unwrap());+ (child, stdin, stdout)+}++fn init_driver(stdin: &mut impl Write, stdout: &mut impl BufRead) {+ send_rpc(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ recv_rpc(stdout);+}++fn call_tool(+ stdin: &mut impl Write,+ stdout: &mut impl BufRead,+ id: u64,+ name: &str,+ args: serde_json::Value,+) -> serde_json::Value {+ send_rpc(stdin, &serde_json::json!({+ "jsonrpc":"2.0","id":id,"method":"tools/call",+ "params":{"name":name,"arguments":args}+ }));+ recv_rpc(stdout)+}++fn tool_ok(resp: &serde_json::Value) -> bool {+ resp["error"].is_null() && !resp["result"]["isError"].as_bool().unwrap_or(false)+}++fn focus_pid_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_pid.txt") }+fn focus_hwnd_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_hwnd.txt") }++/// Launch focus-monitor-win and return (process, hwnd, pid).+/// Reads FOCUS_PID and FOCUS_HWND from temp files written by the monitor+/// (avoids blocking on the stdout pipe if the sandbox redirects I/O).+fn launch_focus_monitor() -> (Child, u64, u32) {+ let exe = focus_monitor_path();+ if !exe.exists() {+ panic!("focus-monitor-win.exe not found at {exe:?} — run `cargo build` first");+ }+ // Reset all sentinel files so stale values are not mistaken for new ones.+ let _ = std::fs::write(loss_file(), "0");+ let _ = std::fs::write(key_loss_file(), "0");+ let _ = std::fs::remove_file(focus_pid_file());+ let _ = std::fs::remove_file(focus_hwnd_file());++ let child = Command::new(&exe)+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn focus-monitor-win");++ // Poll temp files until both PID and HWND are written (max 15s).+ let deadline = std::time::Instant::now() + Duration::from_secs(15);+ let (mut pid_val, mut hwnd_val) = (0u32, 0u64);+ loop {+ if std::time::Instant::now() > deadline {+ panic!("focus-monitor-win did not write PID/HWND temp files within 15s");+ }+ if pid_val == 0 {+ pid_val = std::fs::read_to_string(focus_pid_file())+ .ok().and_then(|s| s.trim().parse().ok()).unwrap_or(0);+ }+ if hwnd_val == 0 {+ hwnd_val = std::fs::read_to_string(focus_hwnd_file())+ .ok().and_then(|s| s.trim().parse().ok()).unwrap_or(0);+ }+ if pid_val != 0 && hwnd_val != 0 { break; }+ std::thread::sleep(Duration::from_millis(100));+ }+ // Give the window time to become foreground.+ std::thread::sleep(Duration::from_millis(400));+ (child, hwnd_val, pid_val)+}++/// Find the first on-screen window belonging to the given pid.+fn find_window_for_pid(+ stdin: &mut impl Write,+ stdout: &mut impl BufRead,+ id: u64,+ pid: i64,+) -> Option<u64> {+ let resp = call_tool(stdin, stdout, id, "list_windows",+ serde_json::json!({"pid": pid, "on_screen_only": true}));+ resp["result"]["structuredContent"]["windows"]+ .as_array()?+ .iter()+ .find_map(|w| w["window_id"].as_u64())+}++// ── UX guard assertion ────────────────────────────────────────────────────────++/// Assert act_losses stayed at `max_allowed` (usually 0) since `before`.+fn assert_ux_guard(before: u32, max_allowed: u32, context: &str) {+ let after = read_losses(&loss_file());+ let delta = after.saturating_sub(before);+ assert!(+ delta <= max_allowed,+ "UX guard violated: act_losses went from {before} to {after} \+ (delta={delta}, max_allowed={max_allowed}) during: {context}"+ );+}++// ─────────────────────────────────────────────────────────────────────────────+// Test 1: background click + type do not steal focus+// ─────────────────────────────────────────────────────────────────────────────++#[test]+fn test_background_click_and_type_no_focus_steal() {+ //! Equivalent of macOS test_background_focus.py.+ //!+ //! 1. Launch FocusMonitorWin (simulates the user's active window).+ //! 2. Launch desktop-test-app-electron in the background.+ //! 3. Click inside the app and type text via cua-driver.+ //! 4. Assert act_losses on FocusMonitorWin stayed at 0.++ let binary = binary_path();+ if !binary.exists() { eprintln!("Binary not found — skipping"); return; }++ let (mut fm_proc, _fm_hwnd, _fm_pid) = launch_focus_monitor();+ let losses_before = read_losses(&loss_file());++ let (mut drv, mut stdin, mut stdout) = spawn_driver();+ init_driver(&mut stdin, &mut stdout);++ let Some((mut app_proc, app_pid)) = launch_test_app() else {+ eprintln!("test app not available — skipping");+ drv.kill().ok(); fm_proc.kill().ok(); return;+ };+ let Some(app_wid) = find_window_for_pid(&mut stdin, &mut stdout, 3, app_pid as i64) else {+ eprintln!("test app window not found — skipping");+ drv.kill().ok(); app_proc.kill().ok(); fm_proc.kill().ok(); return;+ };++ // Click inside the app (background, via PostMessage).+ let r = call_tool(&mut stdin, &mut stdout, 4, "click",+ serde_json::json!({"pid": app_pid, "window_id": app_wid, "x": 200.0, "y": 200.0}));+ assert!(r["error"].is_null(), "Protocol error from click: {r:?}");++ // Type text into the app (background, via PostMessage).+ let r = call_tool(&mut stdin, &mut stdout, 5, "type_text",+ serde_json::json!({"pid": app_pid, "window_id": app_wid, "text": "ux-guard-test"}));+ assert!(r["error"].is_null(), "Protocol error from type_text: {r:?}");++ // ux_guard: FocusMonitorWin must not have lost activation.+ assert_ux_guard(losses_before, 0,+ "background click + type_text into desktop-test-app-electron");++ drv.kill().ok(); app_proc.kill().ok(); fm_proc.kill().ok();+}++// ─────────────────────────────────────────────────────────────────────────────+// Test 2: launch_app does not steal focus+// ─────────────────────────────────────────────────────────────────────────────++#[test]+fn test_launch_app_no_focus_steal() {+ //! Equivalent of macOS test_launch_app_visible.py.+ //!+ //! launch_app (path variant) must open a window without displacing FocusMonitorWin.++ let binary = binary_path();+ if !binary.exists() { return; }++ let exe = test_app_path();+ if !exe.exists() { eprintln!("test app not available — skipping"); return; }++ let (mut fm_proc, _fm_hwnd, _fm_pid) = launch_focus_monitor();+ let losses_before = read_losses(&loss_file());++ let (mut drv, mut stdin, mut stdout) = spawn_driver();+ init_driver(&mut stdin, &mut stdout);++ // Launch the test app via cua-driver launch_app (full path, SW_SHOWNOACTIVATE).+ let path_str = exe.to_string_lossy().into_owned();+ let r = call_tool(&mut stdin, &mut stdout, 2, "launch_app",+ serde_json::json!({"path": path_str}));+ if !tool_ok(&r) {+ eprintln!("launch_app failed — skipping: {:?}", r);+ drv.kill().ok(); fm_proc.kill().ok(); return;+ }++ // Wait for the app window to appear (Electron startup ~2-3s).+ let mut app_pid: Option<i64> = None;+ for _ in 0..20 {+ std::thread::sleep(Duration::from_millis(500));+ let r2 = call_tool(&mut stdin, &mut stdout, 3, "list_apps", serde_json::json!({}));+ if let Some(procs) = r2["result"]["structuredContent"]["processes"].as_array() {+ if let Some(p) = procs.iter().find(|p| {+ p["name"].as_str().map(|n| n.to_lowercase().contains("desktop-test-app")).unwrap_or(false)+ }) {+ app_pid = p["pid"].as_i64();+ break;+ }+ }+ }+ if app_pid.is_none() {+ eprintln!("desktop-test-app not found in process list after launch_app — skipping");+ drv.kill().ok(); fm_proc.kill().ok(); return;+ }++ // ux_guard: FocusMonitorWin must not have lost activation.+ assert_ux_guard(losses_before, 0, "launch_app desktop-test-app-electron");++ // Kill the launched app by exe name.+ Command::new("taskkill").args(["/F", "/T", "/IM", "desktop-test-app-electron.0.1.0.exe"])+ .stdout(Stdio::null()).stderr(Stdio::null()).spawn().ok();+ drv.kill().ok(); fm_proc.kill().ok();+}++// ─────────────────────────────────────────────────────────────────────────────+// Test 3: background hotkey does not steal focus+// ─────────────────────────────────────────────────────────────────────────────++#[test]+fn test_background_hotkey_no_focus_steal() {+ //! Equivalent of macOS test_background_menu_shortcut.py.+ //!+ //! Send Ctrl+A to a background desktop-test-app-electron window.+ //! FocusMonitorWin must never lose activation.++ let binary = binary_path();+ if !binary.exists() { return; }++ let (mut fm_proc, _fm_hwnd, _fm_pid) = launch_focus_monitor();+ let losses_before = read_losses(&loss_file());++ let (mut drv, mut stdin, mut stdout) = spawn_driver();+ init_driver(&mut stdin, &mut stdout);++ let Some((mut app_proc, app_pid)) = launch_test_app() else {+ eprintln!("test app not available — skipping");+ drv.kill().ok(); fm_proc.kill().ok(); return;+ };+ let Some(app_wid) = find_window_for_pid(&mut stdin, &mut stdout, 3, app_pid as i64) else {+ eprintln!("test app window not found — skipping");+ drv.kill().ok(); app_proc.kill().ok(); fm_proc.kill().ok(); return;+ };++ // Send Ctrl+A hotkey to background app (PostMessage, no focus steal).+ let r = call_tool(&mut stdin, &mut stdout, 4, "hotkey",+ serde_json::json!({"pid": app_pid, "window_id": app_wid, "keys": ["ctrl", "a"]}));+ assert!(r["error"].is_null(), "Protocol error from hotkey: {r:?}");++ // ux_guard: FocusMonitorWin must not have lost activation.+ assert_ux_guard(losses_before, 0, "background hotkey ctrl+a to desktop-test-app-electron");++ drv.kill().ok(); app_proc.kill().ok(); fm_proc.kill().ok();+}++// ─────────────────────────────────────────────────────────────────────────────+// Test 4: background click that opens a new window (e.g. File→New dialog)+// ─────────────────────────────────────────────────────────────────────────────++#[test]+fn test_background_click_opens_new_window_focus_preserved() {+ //! Equivalent of macOS test_click_opens_new_window.py.+ //!+ //! 1. FocusMonitorWin is foreground.+ //! 2. Click a link in desktop-test-app-electron — a new window/tab may appear.+ //! 3. FocusMonitorWin must remain active throughout (UX guard).+ //!+ //! Verifies PostMessage doesn't inadvertently activate any new window that+ //! appears as a side-effect of the click.++ let binary = binary_path();+ if !binary.exists() { return; }++ let (mut fm_proc, _fm_hwnd, _fm_pid) = launch_focus_monitor();+ let losses_before = read_losses(&loss_file());++ let (mut drv, mut stdin, mut stdout) = spawn_driver();+ init_driver(&mut stdin, &mut stdout);++ let Some((mut app_proc, app_pid)) = launch_test_app() else {+ eprintln!("test app not available — skipping");+ drv.kill().ok(); fm_proc.kill().ok(); return;+ };+ let Some(app_wid) = find_window_for_pid(&mut stdin, &mut stdout, 3, app_pid as i64) else {+ eprintln!("test app window not found — skipping");+ drv.kill().ok(); app_proc.kill().ok(); fm_proc.kill().ok(); return;+ };++ // Click somewhere in the app content area (may trigger navigation/new window).+ let r = call_tool(&mut stdin, &mut stdout, 4, "click",+ serde_json::json!({"pid": app_pid, "window_id": app_wid, "x": 400.0, "y": 350.0}));+ assert!(r["error"].is_null(), "Protocol error from click: {r:?}");++ // Brief wait for any side-effect windows to appear.+ std::thread::sleep(Duration::from_millis(500));++ // ux_guard: FocusMonitorWin must not have lost activation.+ assert_ux_guard(losses_before, 0,+ "background click in desktop-test-app-electron (may open new window)");++ // Verify FocusMonitorWin is still alive.+ assert!(+ fm_proc.try_wait().expect("try_wait").is_none(),+ "FocusMonitorWin crashed during the test"+ );++ drv.kill().ok(); app_proc.kill().ok(); fm_proc.kill().ok();+}++// ─────────────────────────────────────────────────────────────────────────────+// Test 5: screenshot of background window doesn't steal focus+// ─────────────────────────────────────────────────────────────────────────────++#[test]+fn test_background_screenshot_no_focus_steal() {+ //! PrintWindow captures a background window without activating it.++ let binary = binary_path();+ if !binary.exists() { return; }++ let (mut fm_proc, _fm_hwnd, _fm_pid) = launch_focus_monitor();+ let losses_before = read_losses(&loss_file());++ let (mut drv, mut stdin, mut stdout) = spawn_driver();+ init_driver(&mut stdin, &mut stdout);++ let Some((mut app_proc, app_pid)) = launch_test_app() else {+ eprintln!("test app not available — skipping");+ drv.kill().ok(); fm_proc.kill().ok(); return;+ };+ let Some(app_wid) = find_window_for_pid(&mut stdin, &mut stdout, 3, app_pid as i64) else {+ eprintln!("test app window not found — skipping");+ drv.kill().ok(); app_proc.kill().ok(); fm_proc.kill().ok(); return;+ };++ // Screenshot via PrintWindow — must not activate the window.+ let r = call_tool(&mut stdin, &mut stdout, 4, "screenshot",+ serde_json::json!({"window_id": app_wid}));+ assert!(r["error"].is_null(), "Protocol error from screenshot: {r:?}");++ // ux_guard+ assert_ux_guard(losses_before, 0, "screenshot of background desktop-test-app-electron");++ drv.kill().ok(); app_proc.kill().ok(); fm_proc.kill().ok();+}++// ─────────────────────────────────────────────────────────────────────────────+// Test 6: agent cursor is visually present on screen after move_cursor+// ─────────────────────────────────────────────────────────────────────────────++#[test]+fn test_agent_cursor_visible_on_screen() {+ //! Computer-vision check: after move_cursor the overlay must be visible.+ //!+ //! Steps:+ //! 1. Enable the agent cursor and move it to a known screen position.+ //! 2. Wait for the glide animation to settle (default 750ms).+ //! 3. Capture the screen at the cursor position using screenshot_display_bytes+ //! (BitBlt from display DC — captures layered/overlay windows).+ //! 4. Decode the PNG and sample a 40×40 px patch centred on the cursor.+ //! 5. Assert the patch contains cursor-like pixels (bright or saturated).++ let binary = binary_path();+ if !binary.exists() { eprintln!("Binary not found — skipping"); return; }++ let (mut drv, mut stdin, mut stdout) = spawn_driver();+ init_driver(&mut stdin, &mut stdout);++ // Safe centre-ish position on primary monitor.+ let cx = 640.0_f64;+ let cy = 400.0_f64;++ // Enable cursor overlay and glide to target.+ let r = call_tool(&mut stdin, &mut stdout, 2, "set_agent_cursor_enabled",+ serde_json::json!({"enabled": true}));+ assert!(r["error"].is_null(), "set_agent_cursor_enabled failed: {r:?}");++ let r = call_tool(&mut stdin, &mut stdout, 3, "move_cursor",+ serde_json::json!({"x": cx, "y": cy}));+ assert!(r["error"].is_null(), "move_cursor failed: {r:?}");++ // Wait for the glide animation (750ms default) + a few render frames.+ std::thread::sleep(Duration::from_millis(900));++ // Capture the screen directly (includes layered windows like the overlay).+ let png_bytes = platform_windows::capture::screenshot_display_bytes()+ .expect("screenshot_display_bytes failed");++ drv.kill().ok();++ // Decode PNG.+ let img = image::load_from_memory(&png_bytes).expect("decode PNG");+ let rgba = img.to_rgba8();+ let (iw, ih) = rgba.dimensions();++ // Sample 40×40 patch centred on (cx, cy).+ let half = 20u32;+ let x0 = (cx as u32).saturating_sub(half).min(iw.saturating_sub(1));+ let x1 = (cx as u32 + half).min(iw);+ let y0 = (cy as u32).saturating_sub(half).min(ih.saturating_sub(1));+ let y1 = (cy as u32 + half).min(ih);++ let mut colourful_pixels = 0u32;+ for py in y0..y1 {+ for px in x0..x1 {+ let [r, g, b, a] = rgba.get_pixel(px, py).0;+ if a < 10 { continue; }+ let brightness = r as u32 + g as u32 + b as u32;+ let saturation = r.max(g).max(b) as u32 - r.min(g).min(b) as u32;+ // Accept bright-white stroke pixels OR coloured gradient pixels.+ if brightness > 60 && (saturation > 30 || brightness > 600) {+ colourful_pixels += 1;+ }+ }+ }++ assert!(+ colourful_pixels >= 5,+ "Agent cursor not visible at ({cx},{cy}): only {colourful_pixels} qualifying pixels \+ in 40×40 patch (x={x0}..{x1}, y={y0}..{y1}, image={iw}x{ih}). \+ Overlay may not be rendering or is positioned off-screen."+ );+}diff --git a/libs/cua-driver-rs/crates/cursor-overlay/Cargo.toml b/libs/cua-driver-rs/crates/cursor-overlay/Cargo.tomlnew file mode 100644index 0000000000..4f561f4180--- /dev/null+++ b/libs/cua-driver-rs/crates/cursor-overlay/Cargo.toml@@ -0,0 +1,18 @@+[package]+name = "cursor-overlay"+version.workspace = true+edition.workspace = true++[dependencies]+serde = { workspace = true }+serde_json = { workspace = true }+anyhow = { workspace = true }+tracing = { workspace = true }++# SVG rasterization for --cursor-icon <path.svg>+resvg = { version = "0.44", default-features = false }+tiny-skia = { version = "0.11", default-features = false, features = ["std"] }+usvg = { version = "0.44", default-features = false, features = ["text"] }++# ICO / PNG parsing for --cursor-icon <path.ico|png>+image = { version = "0.25", default-features = false, features = ["png", "jpeg", "ico"] }diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/bezier.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/bezier.rsnew file mode 100644index 0000000000..af1a09a69d--- /dev/null+++ b/libs/cua-driver-rs/crates/cursor-overlay/src/bezier.rs@@ -0,0 +1,134 @@+//! Cubic Bezier math — 1:1 port of `Bezier.swift` and+//! `CursorMotionSegment` from `AgentCursorPathPlanner.cs`.+//!+//! `B(t) = (1-t)³P₀ + 3(1-t)²tP₁ + 3(1-t)t²P₂ + t³P₃`++#[derive(Debug, Clone, Copy, PartialEq)]+pub struct Point {+ pub x: f64,+ pub y: f64,+}++impl Point {+ pub const ZERO: Self = Self { x: 0.0, y: 0.0 };+ pub fn new(x: f64, y: f64) -> Self { Self { x, y } }+ pub fn hypot(self, other: Self) -> f64 {+ let dx = self.x - other.x;+ let dy = self.y - other.y;+ dx.hypot(dy)+ }+}++/// Cubic Bezier segment with cached arc-length.+#[derive(Debug, Clone, Copy)]+pub struct CubicBezier {+ pub start: Point,+ pub control1: Point,+ pub control2: Point,+ pub end: Point,+ /// Arc-length via 32-point numerical integration.+ pub length: f64,+}++impl CubicBezier {+ pub fn new(start: Point, control1: Point, control2: Point, end: Point) -> Self {+ let mut s = Self { start, control1, control2, end, length: 0.0 };+ s.length = s.measure_length();+ s+ }++ /// Point on the curve at `t ∈ [0,1]`.+ pub fn point_at(&self, t: f64) -> Point {+ let t = t.clamp(0.0, 1.0);+ let u = 1.0 - t;+ let (uu, uuu) = (u * u, u * u * u);+ let (tt, ttt) = (t * t, t * t * t);+ Point {+ x: uuu * self.start.x+ + 3.0 * uu * t * self.control1.x+ + 3.0 * u * tt * self.control2.x+ + ttt * self.end.x,+ y: uuu * self.start.y+ + 3.0 * uu * t * self.control1.y+ + 3.0 * u * tt * self.control2.y+ + ttt * self.end.y,+ }+ }++ /// Tangent direction at `t`. Returns a unit-like vector (not normalised).+ pub fn tangent_at(&self, t: f64) -> Point {+ let t = t.clamp(0.0, 1.0);+ let u = 1.0 - t;+ let a = 3.0 * u * u;+ let b = 6.0 * u * t;+ let c = 3.0 * t * t;+ Point {+ x: a * (self.control1.x - self.start.x)+ + b * (self.control2.x - self.control1.x)+ + c * (self.end.x - self.control2.x),+ y: a * (self.control1.y - self.start.y)+ + b * (self.control2.y - self.control1.y)+ + c * (self.end.y - self.control2.y),+ }+ }++ /// Heading in radians at parametric `t`.+ pub fn heading_at(&self, t: f64) -> f64 {+ let tg = self.tangent_at(t);+ tg.y.atan2(tg.x)+ }++ fn measure_length(&self) -> f64 {+ let mut len = 0.0;+ let mut prev = self.start;+ for i in 1..=32 {+ let p = self.point_at(i as f64 / 32.0);+ len += prev.hypot(p);+ prev = p;+ }+ len+ }++ /// Sample at a given arc-distance from start (0..=self.length).+ /// Returns (point, heading_radians).+ pub fn sample_at_distance(&self, dist: f64) -> (Point, f64) {+ if self.length <= 0.0 {+ return (self.end, self.heading_at(1.0));+ }+ let t = (dist / self.length).clamp(0.0, 1.0);+ (self.point_at(t), self.heading_at(t))+ }+}++/// Build a `CubicBezier` from the `CursorMotionPath.Options` knobs.+/// Matches the Swift `CursorMotionPath.init(from:to:options:)` formula.+pub fn build_motion_bezier(+ start: Point,+ end: Point,+ start_handle: f64,+ end_handle: f64,+ arc_size: f64,+ arc_flow: f64,+) -> CubicBezier {+ let dx = end.x - start.x;+ let dy = end.y - start.y;+ let length = dx.hypot(dy).max(1.0);++ // Unit perpendicular (90° CCW of direction vector).+ let perp_x = -dy / length;+ let perp_y = dx / length;++ let deflection = length * arc_size;+ let flow_bias = (arc_flow + 1.0) / 2.0; // map [-1,1] → [0,1]++ let c1_deflect = deflection * (1.0 - 0.5 * flow_bias);+ let c2_deflect = deflection * (1.0 - 0.5 * (1.0 - flow_bias));++ let c1_base = Point::new(start.x + dx * start_handle, start.y + dy * start_handle);+ let c2_base = Point::new(end.x - dx * end_handle, end.y - dy * end_handle);++ let control1 = Point::new(c1_base.x + perp_x * c1_deflect, c1_base.y + perp_y * c1_deflect);+ let control2 = Point::new(c2_base.x + perp_x * c2_deflect, c2_base.y + perp_y * c2_deflect);++ CubicBezier::new(start, control1, control2, end)+}diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/capture_utils.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/capture_utils.rsnew file mode 100644index 0000000000..174a0a0e73--- /dev/null+++ b/libs/cua-driver-rs/crates/cursor-overlay/src/capture_utils.rs@@ -0,0 +1,94 @@+//! Shared image-manipulation utilities for the `zoom` tool.+//!+//! All three platform crates depend on cursor-overlay, so this is the natural+//! home for a pure-Rust crop + JPEG-encode helper.++use image::{DynamicImage, GenericImageView, imageops::FilterType};++/// Result of a crop-and-encode operation; carries coordinate metadata so callers+/// can translate zoom-image pixel coordinates back to full-window pixel coordinates.+pub struct CropResult {+ pub jpeg_bytes: Vec<u8>,+ /// Output image width in pixels.+ pub out_w: u32,+ /// Output image height in pixels.+ pub out_h: u32,+ /// X of the padded crop origin in full-window pixel space.+ pub origin_x: f64,+ /// Y of the padded crop origin in full-window pixel space.+ pub origin_y: f64,+ /// Inverse of the resize scale: `cw / out_w`.+ /// Multiply a zoom-image X coordinate by this to get the full-window offset from origin.+ /// Equal to 1.0 when no downscale was applied.+ pub scale_inv: f64,+}++impl CropResult {+ /// Convert a zoom-image coordinate `(px, py)` back to full-window pixel coordinates.+ pub fn zoom_to_window(&self, px: f64, py: f64) -> (f64, f64) {+ (+ self.origin_x + px * self.scale_inv,+ self.origin_y + py * self.scale_inv,+ )+ }+}++/// Decode a PNG, crop to the given region with 20% padding on all sides,+/// optionally scale down to `max_width`, and re-encode as JPEG.+pub fn crop_png_to_jpeg(+ png_bytes: &[u8],+ x1: f64,+ y1: f64,+ x2: f64,+ y2: f64,+ max_width: u32,+) -> anyhow::Result<CropResult> {+ let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?;+ let (img_w, img_h) = img.dimensions();++ // Region dimensions (at least 1px to avoid divide-by-zero).+ let rw = (x2 - x1).max(1.0);+ let rh = (y2 - y1).max(1.0);++ // 20% outward padding on each side.+ let pad_x = rw * 0.20;+ let pad_y = rh * 0.20;++ let cx1 = ((x1 - pad_x).max(0.0) as u32).min(img_w);+ let cy1 = ((y1 - pad_y).max(0.0) as u32).min(img_h);+ let cx2 = ((x2 + pad_x).min(img_w as f64) as u32).min(img_w);+ let cy2 = ((y2 + pad_y).min(img_h as f64) as u32).min(img_h);++ let cw = cx2.saturating_sub(cx1).max(1);+ let ch = cy2.saturating_sub(cy1).max(1);++ let cropped = img.crop_imm(cx1, cy1, cw, ch);++ // Scale down if the crop is wider than max_width.+ let out: DynamicImage = if cw > max_width {+ let scale = max_width as f64 / cw as f64;+ let new_h = ((ch as f64) * scale).round().max(1.0) as u32;+ cropped.resize_exact(max_width, new_h, FilterType::Lanczos3)+ } else {+ cropped+ };++ let (out_w, out_h) = out.dimensions();+ let scale_inv = cw as f64 / out_w as f64; // 1.0 if no resize++ // Encode as JPEG.+ let mut buf = Vec::new();+ out.write_to(+ &mut std::io::Cursor::new(&mut buf),+ image::ImageFormat::Jpeg,+ )?;++ Ok(CropResult {+ jpeg_bytes: buf,+ out_w,+ out_h,+ origin_x: cx1 as f64,+ origin_y: cy1 as f64,+ scale_inv,+ })+}diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/lib.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/lib.rsnew file mode 100644index 0000000000..b2c6011b1b--- /dev/null+++ b/libs/cua-driver-rs/crates/cursor-overlay/src/lib.rs@@ -0,0 +1,265 @@+//! cursor-overlay — shared types and math for the cua-driver cursor overlay.+//!+//! Platform renderers (macOS, Windows, Linux) depend on this crate for:+//! - `CursorConfig` — parsed from CLI args (`--cursor-icon`, `--cursor-id`, etc.)+//! - `Palette` — 9 named colour palettes matching the reference implementations+//! - `MotionConfig` — glide duration, spring, dwell, idle-hide timings+//! - `CubicBezier` + `PathPlanner` — Bezier path math (ported 1:1 from C#)+//! - `CursorShape` — loaded and rasterised custom SVG / ICO / PNG asset+//! - `OverlayCommand` — messages sent from MCP tools to the overlay thread++pub mod palette;+pub mod motion;+pub mod bezier;+pub mod path_planner;+pub mod shape;+pub mod capture_utils;++pub use palette::Palette;+pub use motion::MotionConfig;+pub use bezier::CubicBezier;+pub use path_planner::{PathPlanner, PlannedPath, PathState};+pub use shape::CursorShape;++/// Configuration assembled from CLI arguments and passed to every+/// platform backend when it initialises the overlay window.+#[derive(Debug, Clone)]+pub struct CursorConfig {+ /// Multi-cursor instance identifier; affects palette selection.+ /// Defaults to `"default"`.+ pub cursor_id: String,++ /// Custom cursor shape loaded from `--cursor-icon <path>`.+ /// `None` means use the built-in gradient arrow.+ pub shape: Option<CursorShape>,++ /// Initial motion config (can be updated at runtime via MCP tool).+ pub motion: MotionConfig,++ /// Whether the overlay is visible at startup.+ /// Pass `--no-overlay` to disable.+ pub enabled: bool,+}++impl Default for CursorConfig {+ fn default() -> Self {+ Self {+ cursor_id: "default".into(),+ shape: None,+ motion: MotionConfig::default(),+ enabled: true,+ }+ }+}++impl CursorConfig {+ /// Parse from `std::env::args()`.+ ///+ /// Recognised flags:+ /// ```text+ /// --cursor-icon <path.svg|path.ico|path.png>+ /// --cursor-id <id>+ /// --cursor-palette <name> (selects a named Palette)+ /// --no-overlay (start with overlay disabled)+ /// --glide-ms <f64> (glideDurationMs override)+ /// --dwell-ms <f64> (dwellAfterClickMs override)+ /// --idle-hide-ms <f64> (idleHideMs override)+ /// ```+ pub fn from_args() -> Self {+ let args: Vec<String> = std::env::args().collect();+ Self::parse(&args[1..])+ }++ pub fn parse(args: &[String]) -> Self {+ let mut cfg = CursorConfig::default();+ let mut i = 0usize;+ while i < args.len() {+ match args[i].as_str() {+ "--cursor-icon" => {+ if let Some(path) = args.get(i + 1) {+ match CursorShape::load(path) {+ Ok(s) => cfg.shape = Some(s),+ Err(e) => tracing::warn!("--cursor-icon {path}: {e}"),+ }+ i += 1;+ }+ }+ "--cursor-id" => {+ if let Some(id) = args.get(i + 1) {+ cfg.cursor_id = id.clone();+ i += 1;+ }+ }+ "--cursor-palette" => {+ if let Some(name) = args.get(i + 1) {+ // Palette is resolved inside the platform backend using the id;+ // store the name as the id so ForInstance logic picks it up.+ cfg.cursor_id = name.clone();+ i += 1;+ }+ }+ "--no-overlay" => cfg.enabled = false,+ "--glide-ms" => {+ if let Some(v) = args.get(i + 1).and_then(|s| s.parse().ok()) {+ cfg.motion.glide_duration_ms = v;+ i += 1;+ }+ }+ "--dwell-ms" => {+ if let Some(v) = args.get(i + 1).and_then(|s| s.parse().ok()) {+ cfg.motion.dwell_after_click_ms = v;+ i += 1;+ }+ }+ "--idle-hide-ms" => {+ if let Some(v) = args.get(i + 1).and_then(|s| s.parse().ok()) {+ cfg.motion.idle_hide_ms = v;+ i += 1;+ }+ }+ _ => {}+ }+ i += 1;+ }+ cfg+ }++ /// Return the `Palette` for this config (by cursor_id).+ pub fn palette(&self) -> Palette {+ Palette::for_instance(&self.cursor_id)+ }+}++// ── Shared cursor instance registry ──────────────────────────────────────────++use serde::{Deserialize, Serialize};+use std::collections::HashMap;+use std::sync::Mutex;++/// Per-instance cursor configuration (icon, color, label, size, opacity).+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct CursorInstanceConfig {+ pub cursor_id: String,+ pub cursor_icon: Option<String>,+ pub cursor_color: Option<String>,+ pub cursor_label: Option<String>,+ pub cursor_size: Option<f64>,+ pub cursor_opacity: Option<f64>,+ pub enabled: bool,+}++impl Default for CursorInstanceConfig {+ fn default() -> Self {+ Self {+ cursor_id: "default".into(),+ cursor_icon: None,+ cursor_color: Some("#00FFFF".into()),+ cursor_label: None,+ cursor_size: Some(16.0),+ cursor_opacity: Some(0.85),+ enabled: true,+ }+ }+}++/// Runtime state for a cursor instance (config + last known position).+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct CursorInstanceState {+ pub config: CursorInstanceConfig,+ pub x: Option<f64>,+ pub y: Option<f64>,+}++/// Global registry of cursor instances, keyed by `cursor_id`.+pub struct CursorRegistry {+ inner: Mutex<HashMap<String, CursorInstanceState>>,+}++impl CursorRegistry {+ pub fn new() -> Self {+ let mut map = HashMap::new();+ map.insert("default".into(), CursorInstanceState {+ config: CursorInstanceConfig::default(),+ x: None,+ y: None,+ });+ Self { inner: Mutex::new(map) }+ }++ pub fn get_or_create(&self, cursor_id: &str) -> CursorInstanceState {+ let mut inner = self.inner.lock().unwrap();+ inner.entry(cursor_id.to_owned()).or_insert_with(|| CursorInstanceState {+ config: CursorInstanceConfig {+ cursor_id: cursor_id.to_owned(), ..Default::default()+ },+ x: None, y: None,+ }).clone()+ }++ pub fn update_position(&self, cursor_id: &str, x: f64, y: f64) {+ let mut inner = self.inner.lock().unwrap();+ let state = inner.entry(cursor_id.to_owned()).or_insert_with(|| CursorInstanceState {+ config: CursorInstanceConfig { cursor_id: cursor_id.to_owned(), ..Default::default() },+ x: None, y: None,+ });+ state.x = Some(x);+ state.y = Some(y);+ }++ pub fn set_enabled(&self, cursor_id: &str, enabled: bool) {+ let mut inner = self.inner.lock().unwrap();+ let state = inner.entry(cursor_id.to_owned()).or_insert_with(|| CursorInstanceState {+ config: CursorInstanceConfig { cursor_id: cursor_id.to_owned(), ..Default::default() },+ x: None, y: None,+ });+ state.config.enabled = enabled;+ }++ pub fn update_config(&self, cursor_id: &str, f: impl FnOnce(&mut CursorInstanceConfig)) {+ let mut inner = self.inner.lock().unwrap();+ let state = inner.entry(cursor_id.to_owned()).or_insert_with(|| CursorInstanceState {+ config: CursorInstanceConfig { cursor_id: cursor_id.to_owned(), ..Default::default() },+ x: None, y: None,+ });+ f(&mut state.config);+ }++ pub fn all_states(&self) -> Vec<CursorInstanceState> {+ self.inner.lock().unwrap().values().cloned().collect()+ }+}++impl Default for CursorRegistry {+ fn default() -> Self { Self::new() }+}++/// Commands sent from MCP tool handlers to the overlay's render thread.+#[derive(Debug, Clone)]+pub enum OverlayCommand {+ /// Animate the cursor to a new screen position.+ MoveTo { x: f64, y: f64, end_heading_radians: f64 },+ /// Start the click-press visual.+ ClickPulse { x: f64, y: f64 },+ /// Show or hide the overlay.+ SetEnabled(bool),+ /// Update the motion/timing config live.+ SetMotion(MotionConfig),+ /// Update the palette live.+ SetPalette(Palette),+ /// Pin the overlay above a specific window (by platform window id).+ PinAbove(u64),+ /// Replace the cursor shape at runtime.+ /// `None` reverts to the built-in gradient arrow.+ SetShape(Option<CursorShape>),+ /// Update the gradient/bloom colours used by the default arrow renderer.+ /// `gradient_colors`: ordered list of `#RRGGBB` hex strings.+ /// `bloom_color`: `#RRGGBB` hex string for the radial halo.+ SetGradient {+ gradient_colors: Vec<[u8; 4]>,+ bloom_color: Option<[u8; 4]>,+ },+ /// Show a focus-highlight rectangle around an AX-targeted element.+ /// `[x, y, width, height]` in screen coordinates (top-left origin).+ /// `None` clears the highlight.+ ShowFocusRect(Option<[f64; 4]>),+}diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/motion.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/motion.rsnew file mode 100644index 0000000000..990d0b0ae3--- /dev/null+++ b/libs/cua-driver-rs/crates/cursor-overlay/src/motion.rs@@ -0,0 +1,84 @@+//! Motion / timing configuration — 1:1 port of `AgentCursorMotion.cs`.++use serde::{Deserialize, Serialize};++/// Runtime-tunable timing and path-shape parameters.+/// All clamp ranges are identical to the C# reference.+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct MotionConfig {+ /// Control-point offset from start, as fraction of distance. [0, 1]+ pub start_handle: f64,+ /// Control-point offset from end. [0, 1]+ pub end_handle: f64,+ /// Perpendicular deflection magnitude as fraction of distance. [0, 1]+ pub arc_size: f64,+ /// Deflection asymmetry: positive = apex near destination. [-1, 1]+ pub arc_flow: f64,+ /// Post-arrival spring damping: 1.0 = critical, 0.3 = bouncy. [0.3, 1.0]+ pub spring: f64,+ /// Main glide duration in milliseconds — used only as a legacy override.+ /// When <= 0 the render engine uses speed-based timing instead. [50, 5000]+ pub glide_duration_ms: f64,+ /// Post-click dwell in milliseconds. [0, 5000]+ pub dwell_after_click_ms: f64,+ /// Auto-hide delay in milliseconds. 0 = never hide. [0, 60000]+ pub idle_hide_ms: f64,+ /// Click-press visual duration. [0, 5000]+ pub press_duration_ms: f64,+ /// Peak cursor speed in pts/sec (speed-based mode). Matches Swift peakSpeed=900.+ pub peak_speed: f64,+ /// Minimum cursor speed at start of glide, pts/sec.+ pub min_start_speed: f64,+ /// Minimum cursor speed at end of glide (deceleration floor), pts/sec.+ pub min_end_speed: f64,+}++impl Default for MotionConfig {+ fn default() -> Self {+ Self {+ start_handle: 0.3,+ end_handle: 0.3,+ arc_size: 0.25,+ arc_flow: 0.0,+ spring: 0.72,+ glide_duration_ms: 0.0, // 0 = speed-based mode+ dwell_after_click_ms: 80.0,+ idle_hide_ms: 20_000.0, // fade 20s after last activity (matches .NET reference)+ press_duration_ms: 120.0,+ peak_speed: 900.0,+ min_start_speed: 300.0,+ min_end_speed: 200.0,+ }+ }+}++impl MotionConfig {+ pub fn with_overrides(+ &self,+ start_handle: Option<f64>,+ end_handle: Option<f64>,+ arc_size: Option<f64>,+ arc_flow: Option<f64>,+ spring: Option<f64>,+ glide_duration_ms: Option<f64>,+ dwell_after_click_ms: Option<f64>,+ idle_hide_ms: Option<f64>,+ press_duration_ms: Option<f64>,+ ) -> Self {+ fn clamp(v: f64, lo: f64, hi: f64) -> f64 { v.clamp(lo, hi) }+ Self {+ start_handle: clamp(start_handle.unwrap_or(self.start_handle), 0.0, 1.0),+ end_handle: clamp(end_handle.unwrap_or(self.end_handle), 0.0, 1.0),+ arc_size: clamp(arc_size.unwrap_or(self.arc_size), 0.0, 1.0),+ arc_flow: clamp(arc_flow.unwrap_or(self.arc_flow), -1.0, 1.0),+ spring: clamp(spring.unwrap_or(self.spring), 0.3, 1.0),+ glide_duration_ms: clamp(glide_duration_ms.unwrap_or(self.glide_duration_ms), 0.0, 5000.0),+ dwell_after_click_ms: clamp(dwell_after_click_ms.unwrap_or(self.dwell_after_click_ms), 0.0, 5000.0),+ idle_hide_ms: clamp(idle_hide_ms.unwrap_or(self.idle_hide_ms), 0.0, 60_000.0),+ press_duration_ms: clamp(press_duration_ms.unwrap_or(self.press_duration_ms), 0.0, 5000.0),+ peak_speed: self.peak_speed,+ min_start_speed: self.min_start_speed,+ min_end_speed: self.min_end_speed,+ }+ }+}diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/palette.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/palette.rsnew file mode 100644index 0000000000..2efc42d1fc--- /dev/null+++ b/libs/cua-driver-rs/crates/cursor-overlay/src/palette.rs@@ -0,0 +1,104 @@+//! Nine named colour palettes — 1:1 port of `AgentCursorPalette.cs`.+//!+//! Colours stored as `[u8; 4]` = [R, G, B, 255].++use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct Palette {+ pub name: String,+ /// Tip colour (lightest, gradient position 0.0).+ pub cursor_start: [u8; 4],+ /// Mid-gradient colour (position 0.53).+ pub cursor_mid: [u8; 4],+ /// Tail colour (position 1.0).+ pub cursor_end: [u8; 4],+ /// Outer bloom layer.+ pub bloom_outer: [u8; 4],+ /// Inner bloom layer (brighter core).+ pub bloom_inner: [u8; 4],+}++// Raw palette data as (&name, start, mid, end, bloom_outer, bloom_inner).+type PaletteData = (&'static str, [u8;4], [u8;4], [u8;4], [u8;4], [u8;4]);++const fn rgba(r: u8, g: u8, b: u8) -> [u8; 4] { [r, g, b, 255] }++const PALETTE_DATA: &[PaletteData] = &[+ ("default_blue", rgba(219,238,255), rgba( 94,192,232), rgba( 84,205,160), rgba(188,232,252), rgba(238,248,255)),+ ("soft_purple", rgba(238,226,255), rgba(178,132,255), rgba(118,194,255), rgba(214,188,255), rgba(246,238,255)),+ ("rose_gold", rgba(255,231,238), rgba(247,132,170), rgba(255,181,108), rgba(255,190,211), rgba(255,243,232)),+ ("mint_lime", rgba(226,255,240), rgba( 96,218,174), rgba(178,229, 72), rgba(178,245,217), rgba(241,255,231)),+ ("amber", rgba(255,244,214), rgba(244,178, 66), rgba(255,126, 92), rgba(255,219,140), rgba(255,248,225)),+ ("aqua", rgba(221,252,255), rgba( 76,204,224), rgba( 63,222,166), rgba(172,241,249), rgba(236,255,251)),+ ("orchid", rgba(252,228,255), rgba(221,113,236), rgba(255,139,196), rgba(237,181,246), rgba(255,239,252)),+ ("crimson", rgba(255,226,226), rgba(232, 82, 98), rgba(150, 94,255), rgba(255,168,178), rgba(255,240,241)),+ ("chartreuse", rgba(247,255,218), rgba(184,220, 54), rgba( 72,190,119), rgba(224,247,128), rgba(249,255,232)),+ ("cobalt", rgba(226,235,255), rgba( 80,126,236), rgba( 91,219,222), rgba(170,195,255), rgba(239,246,255)),+];++fn from_data(d: &PaletteData) -> Palette {+ Palette {+ name: d.0.to_owned(),+ cursor_start: d.1,+ cursor_mid: d.2,+ cursor_end: d.3,+ bloom_outer: d.4,+ bloom_inner: d.5,+ }+}++impl Palette {+ pub fn default_blue() -> Self { from_data(&PALETTE_DATA[0]) }++ /// Select a palette for an instance id using the same stable-hash+ /// logic as the C# `AgentCursorPalette.ForInstance`.+ pub fn for_instance(instance_id: &str) -> Self {+ if instance_id.is_empty() || instance_id == "default" {+ return Self::default_blue();+ }+ // Exact name match.+ if let Some(d) = PALETTE_DATA.iter().find(|d| d.0 == instance_id) {+ return from_data(d);+ }+ // Stable hash into alternates (all except default_blue at index 0).+ let alternates = &PALETTE_DATA[1..];+ let idx = stable_index(instance_id, alternates.len());+ from_data(&alternates[idx])+ }++ pub fn all_names() -> Vec<&'static str> {+ PALETTE_DATA.iter().map(|d| d.0).collect()+ }++ /// Lerp along cursor_start → cursor_mid → cursor_end at `t ∈ [0,1]`.+ pub fn gradient_at(&self, t: f64) -> [u8; 4] {+ let t = t.clamp(0.0, 1.0);+ if t <= 0.53 {+ lerp_rgba(self.cursor_start, self.cursor_mid, t / 0.53)+ } else {+ lerp_rgba(self.cursor_mid, self.cursor_end, (t - 0.53) / 0.47)+ }+ }+}++fn lerp_rgba(a: [u8; 4], b: [u8; 4], t: f64) -> [u8; 4] {+ let l = |x: u8, y: u8| (x as f64 + (y as f64 - x as f64) * t).round() as u8;+ [l(a[0], b[0]), l(a[1], b[1]), l(a[2], b[2]), 255]+}++fn stable_index(id: &str, count: usize) -> usize {+ let suffix = id.rfind(|c| c == '-' || c == '_' || c == '.')+ .map(|i| &id[i + 1..]).unwrap_or(id);+ if let Ok(n) = suffix.parse::<usize>() { if n > 0 { return (n - 1) % count; } }+ if suffix.len() == 1 {+ if let Some(c) = suffix.chars().next() {+ if c.is_ascii_alphabetic() {+ return (c.to_ascii_lowercase() as usize - b'a' as usize) % count;+ }+ }+ }+ let mut hash: u32 = 2_166_136_261;+ for c in id.chars() { hash ^= c as u32; hash = hash.wrapping_mul(16_777_619); }+ (hash as usize) % count+}diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/path_planner.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/path_planner.rsnew file mode 100644index 0000000000..bfbfb75fe2--- /dev/null+++ b/libs/cua-driver-rs/crates/cursor-overlay/src/path_planner.rs@@ -0,0 +1,267 @@+//! Dubins path planner — 1:1 port of the Swift `AgentCursorRenderer` Dubins+//! implementation (`planPath` / `planDubins` / `DubinsPlannedPath`).+//!+//! Plans a minimum-turning-radius arc–straight–arc path from `(x0,y0,th0)`+//! to `(x1,y1,th1)` with turn radius `R`, then samples it at any arc-length.++use std::f64::consts::PI;++// ── Public types ──────────────────────────────────────────────────────────++/// Position + heading at a sampled point on the path.+#[derive(Debug, Clone, Copy)]+pub struct PathState {+ pub x: f64,+ pub y: f64,+ pub heading: f64,+}++/// The kind of path (Dubins arc–straight–arc, or linear fallback).+#[derive(Debug, Clone, Copy, PartialEq)]+enum PathKind { Dubins, Linear }++/// The three Dubins segment types.+#[derive(Debug, Clone, Copy, PartialEq)]+enum SegType { L, R, S }++/// A planned path (Dubins or linear fallback).+#[derive(Debug, Clone)]+pub struct PlannedPath {+ pub length: f64,+ pub end_visual_heading: f64,+ kind: PathKind,+ // Dubins state+ x0: f64, y0: f64, th0: f64,+ r: f64,+ seg1: f64, seg2: f64, seg3: f64,+ types: [SegType; 3],+ // Linear fallback+ x1: f64, y1: f64, th1: f64,+}++impl PlannedPath {+ /// Sample the path at `distance` arc-length from the start.+ /// Matches `DubinsPlannedPath.sample(at:)`.+ pub fn sample(&self, distance: f64) -> PathState {+ match self.kind {+ PathKind::Linear => self.sample_linear(distance),+ PathKind::Dubins => self.sample_dubins(distance),+ }+ }++ fn sample_linear(&self, s: f64) -> PathState {+ let len = self.length.max(1.0);+ let u = (s / len).clamp(0.0, 1.0);+ let mut diff = self.th1 - self.th0;+ while diff > PI { diff -= 2.0 * PI; }+ while diff < -PI { diff += 2.0 * PI; }+ PathState {+ x: self.x0 + (self.x1 - self.x0) * u,+ y: self.y0 + (self.y1 - self.y0) * u,+ heading: self.th0 + diff * u,+ }+ }++ fn sample_dubins(&self, s_in: f64) -> PathState {+ if s_in <= 0.0 {+ return PathState { x: self.x0, y: self.y0, heading: self.th0 };+ }+ let r = self.r;+ let l1 = self.seg1 * r;+ let l2 = self.seg2 * r;+ let l3 = self.seg3 * r;+ let s = s_in.min(l1 + l2 + l3);++ let mut x = self.x0;+ let mut y = self.y0;+ let mut th = self.th0;++ let mut advance = |len: f64, seg: SegType| {+ if seg == SegType::S {+ x += th.cos() * len;+ y += th.sin() * len;+ } else {+ let dth = (len / r) * if seg == SegType::L { 1.0 } else { -1.0 };+ let perp = if seg == SegType::L { PI / 2.0 } else { -PI / 2.0 };+ let cx = x + (th + perp).cos() * r;+ let cy = y + (th + perp).sin() * r;+ let ang = (y - cy).atan2(x - cx);+ x = cx + (ang + dth).cos() * r;+ y = cy + (ang + dth).sin() * r;+ th += dth;+ }+ };++ if s <= l1 {+ advance(s, self.types[0]);+ return PathState { x, y, heading: th };+ }+ advance(l1, self.types[0]);+ if s <= l1 + l2 {+ advance(s - l1, self.types[1]);+ return PathState { x, y, heading: th };+ }+ advance(l2, self.types[1]);+ advance(s - l1 - l2, self.types[2]);+ PathState { x, y, heading: th }+ }+}++// ── PathPlanner entry-point ───────────────────────────────────────────────++pub struct PathPlanner;++impl PathPlanner {+ /// Plan a Dubins cursor path from `(x0,y0)` heading `th0` to `(x1,y1)` heading `th1`.+ ///+ /// * `end_visual_heading` — the arrow heading at rest (for the rendered tip).+ /// * `turn_radius` — minimum turning radius in points (Swift default: 80).+ pub fn plan(+ x0: f64, y0: f64, th0: f64,+ x1: f64, y1: f64, th1: f64,+ end_visual_heading: f64,+ turn_radius: f64,+ ) -> PlannedPath {+ let r = turn_radius.max(1.0);+ if let Some(p) = plan_dubins(x0, y0, th0, x1, y1, th1, r, end_visual_heading) {+ return p;+ }+ // Linear fallback.+ let d = (x1 - x0).hypot(y1 - y0).max(1.0);+ PlannedPath {+ length: d,+ end_visual_heading,+ kind: PathKind::Linear,+ x0, y0, th0, r,+ seg1: 0.0, seg2: 0.0, seg3: 0.0,+ types: [SegType::S, SegType::S, SegType::S],+ x1, y1, th1,+ }+ }+}++// ── Dubins internals ──────────────────────────────────────────────────────++fn mod2pi(x: f64) -> f64 {+ let tau = 2.0 * PI;+ let r = x - tau * (x / tau).floor();+ if r < 0.0 { r + tau } else { r }+}++struct DubinsSol {+ t: f64, p: f64, q: f64,+ types: [SegType; 3],+}++impl DubinsSol {+ fn length(&self) -> f64 { self.t + self.p + self.q }+}++fn dubins_lsl(d: f64, a: f64, b: f64) -> Option<DubinsSol> {+ let tmp0 = d + a.sin() - b.sin();+ let p2 = 2.0 + d * d - 2.0 * (a - b).cos() + 2.0 * d * (a.sin() - b.sin());+ if p2 < 0.0 { return None; }+ let tmp1 = (b.cos() - a.cos()).atan2(tmp0);+ Some(DubinsSol {+ t: mod2pi(-a + tmp1), p: p2.sqrt(), q: mod2pi(b - tmp1),+ types: [SegType::L, SegType::S, SegType::L],+ })+}++fn dubins_rsr(d: f64, a: f64, b: f64) -> Option<DubinsSol> {+ let tmp0 = d - a.sin() + b.sin();+ let p2 = 2.0 + d * d - 2.0 * (a - b).cos() + 2.0 * d * (b.sin() - a.sin());+ if p2 < 0.0 { return None; }+ let tmp1 = (a.cos() - b.cos()).atan2(tmp0);+ Some(DubinsSol {+ t: mod2pi(a - tmp1), p: p2.sqrt(), q: mod2pi(-b + tmp1),+ types: [SegType::R, SegType::S, SegType::R],+ })+}++fn dubins_lsr(d: f64, a: f64, b: f64) -> Option<DubinsSol> {+ let p2 = -2.0 + d * d + 2.0 * (a - b).cos() + 2.0 * d * (a.sin() + b.sin());+ if p2 < 0.0 { return None; }+ let p = p2.sqrt();+ let tmp1 = (-(a.cos() + b.cos())).atan2(d + a.sin() + b.sin()) - (-2.0_f64).atan2(p);+ Some(DubinsSol {+ t: mod2pi(-a + tmp1), p, q: mod2pi(-mod2pi(b) + tmp1),+ types: [SegType::L, SegType::S, SegType::R],+ })+}++fn dubins_rsl(d: f64, a: f64, b: f64) -> Option<DubinsSol> {+ let p2 = d * d - 2.0 + 2.0 * (a - b).cos() - 2.0 * d * (a.sin() + b.sin());+ if p2 < 0.0 { return None; }+ let p = p2.sqrt();+ let tmp1 = (a.cos() + b.cos()).atan2(d - a.sin() - b.sin()) - 2.0_f64.atan2(p);+ Some(DubinsSol {+ t: mod2pi(a - tmp1), p, q: mod2pi(b - tmp1),+ types: [SegType::R, SegType::S, SegType::L],+ })+}++fn dubins_rlr(d: f64, a: f64, b: f64) -> Option<DubinsSol> {+ let tmp = (6.0 - d * d + 2.0 * (a - b).cos() + 2.0 * d * (a.sin() - b.sin())) / 8.0;+ if tmp.abs() > 1.0 { return None; }+ let p = mod2pi(2.0 * PI - tmp.acos());+ let t = mod2pi(a - (a.cos() - b.cos()).atan2(d - a.sin() + b.sin()) + p / 2.0);+ Some(DubinsSol {+ t, p, q: mod2pi(a - b - t + p),+ types: [SegType::R, SegType::L, SegType::R],+ })+}++fn dubins_lrl(d: f64, a: f64, b: f64) -> Option<DubinsSol> {+ let tmp = (6.0 - d * d + 2.0 * (a - b).cos() + 2.0 * d * (b.sin() - a.sin())) / 8.0;+ if tmp.abs() > 1.0 { return None; }+ let p = mod2pi(2.0 * PI - tmp.acos());+ let t = mod2pi(-a + (-a.cos() + b.cos()).atan2(d + a.sin() - b.sin()) + p / 2.0);+ Some(DubinsSol {+ t, p, q: mod2pi(mod2pi(b) - a - t + p),+ types: [SegType::L, SegType::R, SegType::L],+ })+}++fn plan_dubins(+ x0: f64, y0: f64, th0: f64,+ x1: f64, y1: f64, th1: f64,+ r: f64,+ end_visual_heading: f64,+) -> Option<PlannedPath> {+ let dx = x1 - x0;+ let dy = y1 - y0;+ let d_dist = dx.hypot(dy);+ if d_dist < 0.5 { return None; }+ let d = d_dist / r;+ let theta = mod2pi(dy.atan2(dx));+ let a = mod2pi(th0 - theta);+ let b = mod2pi(th1 - theta);++ let solvers: [fn(f64, f64, f64) -> Option<DubinsSol>; 6] = [+ dubins_lsl, dubins_rsr, dubins_lsr, dubins_rsl, dubins_rlr, dubins_lrl,+ ];++ let mut best_len = f64::INFINITY;+ let mut best: Option<DubinsSol> = None;+ for solver in &solvers {+ if let Some(sol) = solver(d, a, b) {+ let len = sol.length();+ if len.is_finite() && len >= 0.0 && len < best_len {+ best_len = len;+ best = Some(sol);+ }+ }+ }++ let sol = best?;+ Some(PlannedPath {+ length: (sol.t + sol.p + sol.q) * r,+ end_visual_heading,+ kind: PathKind::Dubins,+ x0, y0, th0, r,+ seg1: sol.t, seg2: sol.p, seg3: sol.q,+ types: sol.types,+ x1, y1, th1,+ })+}diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/shape.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/shape.rsnew file mode 100644index 0000000000..749ed5aed5--- /dev/null+++ b/libs/cua-driver-rs/crates/cursor-overlay/src/shape.rs@@ -0,0 +1,78 @@+//! Custom cursor shape — rasterised from SVG / ICO / PNG.+//!+//! Used when `--cursor-icon <path>` is passed to the MCP binary.+//! Always produces a 64×64 RGBA pixel buffer.++use anyhow::{bail, Result};++/// Rasterised cursor shape at 64×64 RGBA.+#[derive(Debug, Clone)]+pub struct CursorShape {+ /// Raw RGBA pixels, row-major top-to-bottom, 4 bytes per pixel.+ pub pixels: Vec<u8>,+ pub width: u32,+ pub height: u32,+}++pub const CURSOR_SIZE: u32 = 64;++impl CursorShape {+ /// Load from `path`. Supported: `.svg`, `.ico`, `.png`.+ pub fn load(path: &str) -> Result<Self> {+ let lower = path.to_ascii_lowercase();+ if lower.ends_with(".svg") {+ Self::load_svg(path)+ } else if lower.ends_with(".ico") || lower.ends_with(".png") {+ Self::load_raster(path)+ } else {+ bail!("Unsupported cursor-icon format (expected .svg, .ico, or .png): {path}")+ }+ }++ fn load_svg(path: &str) -> Result<Self> {+ let data = std::fs::read(path)?;+ let opts = usvg::Options::default();+ let tree = usvg::Tree::from_data(&data, &opts)?;++ let mut pixmap = tiny_skia::Pixmap::new(CURSOR_SIZE, CURSOR_SIZE)+ .ok_or_else(|| anyhow::anyhow!("Failed to create {CURSOR_SIZE}×{CURSOR_SIZE} pixmap"))?;++ let sx = CURSOR_SIZE as f32 / tree.size().width();+ let sy = CURSOR_SIZE as f32 / tree.size().height();+ resvg::render(&tree, tiny_skia::Transform::from_scale(sx, sy), &mut pixmap.as_mut());++ // tiny-skia gives pre-multiplied RGBA; un-premultiply for straight-alpha consumers.+ let raw = pixmap.take();+ let pixels = unpremultiply(raw);+ Ok(Self { pixels, width: CURSOR_SIZE, height: CURSOR_SIZE })+ }++ fn load_raster(path: &str) -> Result<Self> {+ let img = image::open(path)?.into_rgba8();+ let resized = image::imageops::resize(+ &img,+ CURSOR_SIZE,+ CURSOR_SIZE,+ image::imageops::FilterType::Lanczos3,+ );+ Ok(Self {+ pixels: resized.into_raw(),+ width: CURSOR_SIZE,+ height: CURSOR_SIZE,+ })+ }+}++/// Convert pre-multiplied RGBA → straight RGBA.+fn unpremultiply(mut data: Vec<u8>) -> Vec<u8> {+ for px in data.chunks_exact_mut(4) {+ let a = px[3];+ if a > 0 && a < 255 {+ let scale = 255.0 / a as f32;+ px[0] = (px[0] as f32 * scale).min(255.0) as u8;+ px[1] = (px[1] as f32 * scale).min(255.0) as u8;+ px[2] = (px[2] as f32 * scale).min(255.0) as u8;+ }+ }+ data+}diff --git a/libs/cua-driver-rs/crates/focus-monitor-win/Cargo.toml b/libs/cua-driver-rs/crates/focus-monitor-win/Cargo.tomlnew file mode 100644index 0000000000..fbe73e08ef--- /dev/null+++ b/libs/cua-driver-rs/crates/focus-monitor-win/Cargo.toml@@ -0,0 +1,18 @@+[package]+name = "focus-monitor-win"+version.workspace = true+edition.workspace = true+# Windows-only binary used by integration tests to verify the UX guard:+# clicking/typing into background windows must not steal focus from this window.++[[bin]]+name = "focus-monitor-win"+path = "src/main.rs"++[target.'cfg(target_os = "windows")'.dependencies]+windows = { version = "0.58", features = [+ "Win32_Foundation",+ "Win32_UI_WindowsAndMessaging",+ "Win32_System_Threading",+ "Win32_Graphics_Gdi",+] }diff --git a/libs/cua-driver-rs/crates/focus-monitor-win/src/main.rs b/libs/cua-driver-rs/crates/focus-monitor-win/src/main.rsnew file mode 100644index 0000000000..1e55bdbd12--- /dev/null+++ b/libs/cua-driver-rs/crates/focus-monitor-win/src/main.rs@@ -0,0 +1,150 @@+/// focus-monitor-win — Windows equivalent of macOS FocusMonitorApp.+///+/// Creates a visible Win32 window and tracks three kinds of focus loss:+///+/// 1. WM_ACTIVATE (wParam==WA_INACTIVE): the window loses activation.+/// Written to %TEMP%\focus_monitor_losses.txt+///+/// 2. WM_KILLFOCUS: the window loses keyboard focus.+/// Written to %TEMP%\focus_monitor_key_losses.txt+///+/// Prints FOCUS_PID=<pid> on stdout at startup so the test harness can+/// discover the process, then prints FOCUS_HWND=<hwnd_as_u64> so tests+/// can target it with cua-driver tools.+///+/// Exits cleanly on WM_DESTROY.++#[cfg(not(target_os = "windows"))]+fn main() {+ eprintln!("focus-monitor-win is Windows-only.");+ std::process::exit(1);+}++#[cfg(target_os = "windows")]+mod win {+ use std::sync::atomic::{AtomicU32, Ordering};+ use std::ffi::OsStr;+ use std::os::windows::ffi::OsStrExt;+ use windows::Win32::Foundation::*;+ use windows::Win32::Graphics::Gdi::*;+ use windows::Win32::System::Threading::GetCurrentProcessId;+ use windows::Win32::UI::WindowsAndMessaging::*;++ // ── global loss counters ─────────────────────────────────────────────────+ static ACTIVATE_LOSSES: AtomicU32 = AtomicU32::new(0);+ static KEY_LOSSES: AtomicU32 = AtomicU32::new(0);++ fn loss_file() -> std::path::PathBuf { loss_path("focus_monitor_losses.txt") }+ fn key_loss_file() -> std::path::PathBuf { loss_path("focus_monitor_key_losses.txt") }++ fn loss_path(name: &str) -> std::path::PathBuf {+ let mut p = std::env::temp_dir();+ p.push(name);+ p+ }++ fn write_count(path: &std::path::Path, n: u32) {+ let _ = std::fs::write(path, n.to_string());+ }++ fn wide(s: &str) -> Vec<u16> {+ OsStr::new(s).encode_wide().chain(std::iter::once(0)).collect()+ }++ unsafe extern "system" fn wnd_proc(+ hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM,+ ) -> LRESULT {+ match msg {+ WM_ACTIVATE => {+ // WA_INACTIVE == 0 in the low word of wParam+ if (wparam.0 & 0xFFFF) == 0 {+ let n = ACTIVATE_LOSSES.fetch_add(1, Ordering::SeqCst) + 1;+ write_count(&loss_file(), n);+ // Repaint to show updated count+ let _ = InvalidateRect(hwnd, None, true);+ }+ }+ WM_KILLFOCUS => {+ let n = KEY_LOSSES.fetch_add(1, Ordering::SeqCst) + 1;+ write_count(&key_loss_file(), n);+ let _ = InvalidateRect(hwnd, None, true);+ }+ WM_PAINT => {+ let mut ps = PAINTSTRUCT::default();+ let hdc = BeginPaint(hwnd, &mut ps);+ let act = ACTIVATE_LOSSES.load(Ordering::SeqCst);+ let key = KEY_LOSSES.load(Ordering::SeqCst);+ let text = wide(&format!(+ "act_losses: {act} key_losses: {key} (should stay 0)"+ ));+ TextOutW(hdc, 10, 10, &text);+ EndPaint(hwnd, &ps);+ }+ WM_DESTROY => {+ PostQuitMessage(0);+ }+ _ => return DefWindowProcW(hwnd, msg, wparam, lparam),+ }+ LRESULT(0)+ }++ pub fn run() {+ unsafe {+ let class_name = wide("FocusMonitorWin");++ let wc = WNDCLASSW {+ lpfnWndProc: Some(wnd_proc),+ hInstance: HINSTANCE(std::ptr::null_mut()),+ lpszClassName: windows::core::PCWSTR(class_name.as_ptr()),+ hbrBackground: HBRUSH(COLOR_WINDOW.0 as *mut _),+ ..Default::default()+ };+ RegisterClassW(&wc);++ let title = wide("Focus Monitor (cua-driver UX guard)");+ let hwnd = CreateWindowExW(+ WINDOW_EX_STYLE(0),+ windows::core::PCWSTR(class_name.as_ptr()),+ windows::core::PCWSTR(title.as_ptr()),+ WS_OVERLAPPEDWINDOW,+ 100, 100, 600, 140,+ None, None,+ HINSTANCE(std::ptr::null_mut()),+ None,+ ).expect("CreateWindowExW failed");++ ShowWindow(hwnd, SW_SHOWNORMAL);+ UpdateWindow(hwnd).ok();++ // Write initial zeros so tests can read even before any loss.+ write_count(&loss_file(), 0);+ write_count(&key_loss_file(), 0);++ // Signal the test harness via temp files (avoids pipe-blocking issues+ // when stdout is captured by the test runner in sandbox environments).+ let pid = GetCurrentProcessId();+ let hwnd_val = hwnd.0 as usize;+ let pid_file = std::env::temp_dir().join("focus_monitor_pid.txt");+ let hwnd_file = std::env::temp_dir().join("focus_monitor_hwnd.txt");+ let _ = std::fs::write(&pid_file, pid.to_string());+ let _ = std::fs::write(&hwnd_file, hwnd_val.to_string());+ // Also print to stdout as a secondary signal.+ println!("FOCUS_PID={pid}");+ println!("FOCUS_HWND={hwnd_val}");+ use std::io::Write;+ std::io::stdout().flush().ok();++ // Message loop.+ let mut msg = MSG::default();+ while GetMessageW(&mut msg, None, 0, 0).as_bool() {+ TranslateMessage(&msg);+ DispatchMessageW(&msg);+ }+ }+ }+}++#[cfg(target_os = "windows")]+fn main() {+ win::run();+}diff --git a/libs/cua-driver-rs/crates/mcp-server/Cargo.toml b/libs/cua-driver-rs/crates/mcp-server/Cargo.tomlnew file mode 100644index 0000000000..3594790e24--- /dev/null+++ b/libs/cua-driver-rs/crates/mcp-server/Cargo.toml@@ -0,0 +1,13 @@+[package]+name = "mcp-server"+version.workspace = true+edition.workspace = true++[dependencies]+tokio = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+anyhow = { workspace = true }+thiserror = { workspace = true }+tracing = { workspace = true }+async-trait = "0.1"diff --git a/libs/cua-driver-rs/crates/mcp-server/src/browser_eval.rs b/libs/cua-driver-rs/crates/mcp-server/src/browser_eval.rsnew file mode 100644index 0000000000..7cea2b52d1--- /dev/null+++ b/libs/cua-driver-rs/crates/mcp-server/src/browser_eval.rs@@ -0,0 +1,397 @@+//! `browser_eval` tool — Chromium CDP Runtime.evaluate over a local WebSocket.+//!+//! Cross-platform: all I/O is plain TCP to localhost (no external HTTP/WS crates).+//! Mirrors the Windows C# reference in CuaDriver.Win/Tools/BrowserEvalTool.cs+//! and CuaDriver.Win/Browser/CdpBrowserBridge.cs.+//!+//! Flow:+//! 1. HTTP GET http://127.0.0.1:{cdp_port}/json → discover page WebSocket URLs.+//! 2. WebSocket connect to the first (or best-matched) page URL.+//! 3. Send Runtime.evaluate with userGesture=true.+//! 4. Read frames until we receive the response with id=1.++use async_trait::async_trait;+use serde_json::Value;++use crate::{+ protocol::ToolResult,+ tool::{Tool, ToolDef},+};++pub struct BrowserEvalTool;++#[async_trait]+impl Tool for BrowserEvalTool {+ fn def(&self) -> &ToolDef {+ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+ DEF.get_or_init(|| ToolDef {+ name: "browser_eval".into(),+ description: "Evaluate JavaScript in a Chromium browser via CDP Runtime.evaluate \+ with userGesture=true. Requires cdp_port. Pass window_id to target \+ a specific browser window/tab (matched by title heuristic).".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["expression"],+ "properties": {+ "expression": {+ "type": "string",+ "description": "JavaScript expression to evaluate."+ },+ "cdp_port": {+ "type": "integer",+ "description": "Chromium remote debugging port (--remote-debugging-port)."+ },+ "window_id": {+ "type": "integer",+ "description": "Optional browser window/HWND/XID used to bind CDP to the matching tab."+ },+ "await_promise": {+ "type": "boolean",+ "description": "Await a returned Promise (default true)."+ }+ }+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let expression = match args.get("expression").and_then(|v| v.as_str()) {+ Some(e) => e.to_string(),+ None => return ToolResult::error("browser_eval: 'expression' is required"),+ };+ let cdp_port = match args.get("cdp_port").and_then(|v| v.as_u64()) {+ Some(p) if p > 0 && p < 65536 => p as u16,+ _ => return ToolResult::error(+ "browser_eval: 'cdp_port' is required (Chromium --remote-debugging-port value)"+ ),+ };+ let await_promise = args.get("await_promise")+ .and_then(|v| v.as_bool())+ .unwrap_or(true);+ let window_title_hint = args.get("window_id").map(|_| ""); // reserved for future matching++ match cdp_evaluate(cdp_port, window_title_hint, &expression, await_promise).await {+ Ok(response) => {+ let result_text = format_cdp_result(&response);+ ToolResult::text(format!("cdp.runtime.evaluate.user_gesture: {result_text}"))+ }+ Err(e) => ToolResult::error(format!("browser_eval: {e}")),+ }+ }+}++// ── High-level evaluate ───────────────────────────────────────────────────++async fn cdp_evaluate(+ port: u16,+ _window_title_hint: Option<&str>,+ expression: &str,+ await_promise: bool,+) -> anyhow::Result<Value> {+ // Discover page tabs.+ let pages = cdp_list_pages(port).await?;+ if pages.is_empty() {+ anyhow::bail!("No CDP page tabs found on port {port}");+ }++ // Take the first available page (future: match by window title hint).+ let ws_url = pages[0]+ .get("webSocketDebuggerUrl")+ .and_then(|u| u.as_str())+ .ok_or_else(|| anyhow::anyhow!("Page has no webSocketDebuggerUrl"))?+ .to_string();++ let cmd = serde_json::json!({+ "id": 1,+ "method": "Runtime.evaluate",+ "params": {+ "expression": expression,+ "userGesture": true,+ "awaitPromise": await_promise+ }+ });++ // 30-second wall-clock timeout (covers awaited promises).+ tokio::time::timeout(+ std::time::Duration::from_secs(30),+ cdp_ws_call(port, &ws_url, &cmd.to_string()),+ )+ .await+ .map_err(|_| anyhow::anyhow!("CDP evaluate timed out after 30 s"))?+}++fn format_cdp_result(response: &Value) -> String {+ // If the result contains an exception, surface it.+ if let Some(exc) = response+ .get("result")+ .and_then(|r| r.get("exceptionDetails"))+ {+ let msg = exc+ .get("exception")+ .and_then(|e| e.get("description"))+ .and_then(|d| d.as_str())+ .unwrap_or_else(|| exc.to_string().leak()); // static lifetime for error msg+ return format!("exception: {msg}");+ }++ // Extract the evaluated value.+ if let Some(result) = response.get("result").and_then(|r| r.get("result")) {+ if let Some(v) = result.get("value") {+ return v.to_string();+ }+ if let Some(desc) = result.get("description").and_then(|d| d.as_str()) {+ return desc.to_string();+ }+ }++ response.to_string()+}++// ── HTTP page discovery ───────────────────────────────────────────────────++/// GET http://127.0.0.1:{port}/json → array of page descriptors.+/// Returns only entries whose `type` == "page" and that have a webSocketDebuggerUrl.+async fn cdp_list_pages(port: u16) -> anyhow::Result<Vec<Value>> {+ use tokio::io::{AsyncReadExt, AsyncWriteExt};++ let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port)).await+ .map_err(|e| anyhow::anyhow!("Cannot connect to CDP on port {port}: {e}"))?;++ let req = format!(+ "GET /json HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n"+ );+ stream.write_all(req.as_bytes()).await?;++ let mut buf = Vec::new();+ stream.read_to_end(&mut buf).await?;++ let text = std::str::from_utf8(&buf)+ .map_err(|_| anyhow::anyhow!("CDP /json response is not valid UTF-8"))?;+ let body_start = text+ .find("\r\n\r\n")+ .ok_or_else(|| anyhow::anyhow!("Invalid HTTP response from CDP"))? + 4;++ let all: Value = serde_json::from_str(&text[body_start..])+ .map_err(|e| anyhow::anyhow!("CDP /json parse error: {e}"))?;++ let pages = all+ .as_array()+ .ok_or_else(|| anyhow::anyhow!("CDP /json is not a JSON array"))?+ .iter()+ .filter(|p| {+ p.get("type").and_then(|t| t.as_str()) == Some("page")+ && p.get("webSocketDebuggerUrl")+ .and_then(|u| u.as_str())+ .is_some()+ })+ .cloned()+ .collect();++ Ok(pages)+}++// ── WebSocket CDP call ────────────────────────────────────────────────────++/// Open a WebSocket to `ws_url` (on `port`), send `message`, and return the+/// first response frame whose JSON `id` field equals 1.+/// Intermediate event notifications (no `id` field or id != 1) are silently skipped.+async fn cdp_ws_call(port: u16, ws_url: &str, message: &str) -> anyhow::Result<Value> {+ use tokio::io::{AsyncReadExt, AsyncWriteExt};++ // Extract the URL path from e.g. "ws://127.0.0.1:9222/devtools/page/ABC".+ let path: String = if let Some(rest) = ws_url.strip_prefix("ws://") {+ rest.splitn(2, '/')+ .nth(1)+ .map(|p| format!("/{p}"))+ .unwrap_or_else(|| "/".into())+ } else {+ ws_url.to_string()+ };++ let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port)).await+ .map_err(|e| anyhow::anyhow!("Cannot connect to CDP WebSocket on port {port}: {e}"))?;++ // WebSocket HTTP upgrade.+ // The Sec-WebSocket-Key is a static value — fine for localhost non-security use.+ let handshake = format!(+ "GET {path} HTTP/1.1\r\n\+ Host: 127.0.0.1:{port}\r\n\+ Upgrade: websocket\r\n\+ Connection: Upgrade\r\n\+ Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\+ Sec-WebSocket-Version: 13\r\n\+ \r\n"+ );+ stream.write_all(handshake.as_bytes()).await?;++ // Read until end of HTTP response headers.+ let mut hdr = Vec::with_capacity(512);+ loop {+ let mut b = [0u8; 1];+ stream.read_exact(&mut b).await?;+ hdr.push(b[0]);+ if hdr.ends_with(b"\r\n\r\n") {+ break;+ }+ if hdr.len() > 8192 {+ anyhow::bail!("WebSocket upgrade response headers too large");+ }+ }+ if !String::from_utf8_lossy(&hdr).contains("101") {+ anyhow::bail!(+ "WebSocket upgrade failed: {}",+ String::from_utf8_lossy(&hdr[..hdr.len().min(200)])+ );+ }++ // Send the CDP command as a masked WebSocket text frame.+ ws_write_text(&mut stream, message.as_bytes()).await?;++ // Read frames, skipping events, until we receive id=1.+ loop {+ match ws_read_frame(&mut stream).await? {+ WsFrame::Text(text) => {+ // Try to parse as JSON and check for id == 1.+ if let Ok(val) = serde_json::from_str::<Value>(&text) {+ if val.get("id").and_then(|id| id.as_u64()) == Some(1) {+ return Ok(val);+ }+ // Event notification — ignore and loop.+ } else {+ anyhow::bail!("CDP returned non-JSON frame: {}", &text[..text.len().min(120)]);+ }+ }+ WsFrame::Ping(data) => {+ // Respond with pong (RFC 6455 §5.5.3).+ ws_write_pong(&mut stream, &data).await?;+ }+ WsFrame::Close => {+ anyhow::bail!("CDP WebSocket closed by server before response");+ }+ WsFrame::Other => { /* binary / continuation / pong — skip */ }+ }+ }+}++// ── WebSocket frame I/O ───────────────────────────────────────────────────++enum WsFrame {+ Text(String),+ Ping(Vec<u8>),+ Close,+ Other,+}++/// Read one complete (non-fragmented) WebSocket frame.+async fn ws_read_frame(stream: &mut tokio::net::TcpStream) -> anyhow::Result<WsFrame> {+ use tokio::io::AsyncReadExt;++ // First 2 bytes: FIN+RSV+opcode, MASK+payload_len.+ let mut hdr = [0u8; 2];+ stream.read_exact(&mut hdr).await?;++ let opcode = hdr[0] & 0x0F;+ let masked = (hdr[1] & 0x80) != 0;+ let len_byte = (hdr[1] & 0x7F) as usize;++ let payload_len: usize = match len_byte {+ 0..=125 => len_byte,+ 126 => {+ let mut b = [0u8; 2];+ stream.read_exact(&mut b).await?;+ u16::from_be_bytes(b) as usize+ }+ _ => {+ let mut b = [0u8; 8];+ stream.read_exact(&mut b).await?;+ // Guard against absurdly large CDP messages (>16 MiB).+ let n = u64::from_be_bytes(b) as usize;+ if n > 16 * 1024 * 1024 {+ anyhow::bail!("CDP WebSocket frame too large ({n} bytes)");+ }+ n+ }+ };++ let mask_key: Option<[u8; 4]> = if masked {+ let mut m = [0u8; 4];+ stream.read_exact(&mut m).await?;+ Some(m)+ } else {+ None+ };++ let mut payload = vec![0u8; payload_len];+ stream.read_exact(&mut payload).await?;+ if let Some(mk) = mask_key {+ for (i, b) in payload.iter_mut().enumerate() {+ *b ^= mk[i % 4];+ }+ }++ match opcode {+ 1 => Ok(WsFrame::Text(+ String::from_utf8(payload)+ .map_err(|_| anyhow::anyhow!("CDP returned non-UTF-8 text frame"))?,+ )),+ 2 => Ok(WsFrame::Other), // binary — CDP doesn't use this+ 8 => Ok(WsFrame::Close),+ 9 => Ok(WsFrame::Ping(payload)),+ _ => Ok(WsFrame::Other),+ }+}++/// Write a masked WebSocket text frame (client → server, RFC 6455 §5.3 requires masking).+async fn ws_write_text(stream: &mut tokio::net::TcpStream, payload: &[u8]) -> anyhow::Result<()> {+ use tokio::io::AsyncWriteExt;+ // Fixed mask key — fine for a local loopback connection.+ const MASK: [u8; 4] = [0xDE, 0xAD, 0xBE, 0xEF];+ let len = payload.len();++ let mut frame = Vec::with_capacity(len + 10);+ frame.push(0x81u8); // FIN=1, opcode=TEXT(1)+ encode_len_masked(&mut frame, len);+ frame.extend_from_slice(&MASK);+ for (i, &b) in payload.iter().enumerate() {+ frame.push(b ^ MASK[i % 4]);+ }+ stream.write_all(&frame).await?;+ Ok(())+}++/// Write a masked WebSocket pong frame in response to a ping.+async fn ws_write_pong(stream: &mut tokio::net::TcpStream, payload: &[u8]) -> anyhow::Result<()> {+ use tokio::io::AsyncWriteExt;+ const MASK: [u8; 4] = [0xCA, 0xFE, 0xBA, 0xBE];+ let len = payload.len().min(125); // control frames ≤ 125 bytes (RFC 6455 §5.5)++ let mut frame = Vec::with_capacity(len + 6);+ frame.push(0x8Au8); // FIN=1, opcode=PONG(0xA)+ frame.push(0x80 | len as u8); // MASK bit + length+ frame.extend_from_slice(&MASK);+ for (i, &b) in payload[..len].iter().enumerate() {+ frame.push(b ^ MASK[i % 4]);+ }+ stream.write_all(&frame).await?;+ Ok(())+}++/// Encode a (masked) length field per RFC 6455 §5.2.+fn encode_len_masked(buf: &mut Vec<u8>, len: usize) {+ if len < 126 {+ buf.push(0x80 | len as u8);+ } else if len <= 65535 {+ buf.push(0x80 | 126);+ buf.push((len >> 8) as u8);+ buf.push(len as u8);+ } else {+ buf.push(0x80 | 127);+ for i in (0..8).rev() {+ buf.push((len >> (i * 8)) as u8);+ }+ }+}diff --git a/libs/cua-driver-rs/crates/mcp-server/src/lib.rs b/libs/cua-driver-rs/crates/mcp-server/src/lib.rsnew file mode 100644index 0000000000..5815e7e9f5--- /dev/null+++ b/libs/cua-driver-rs/crates/mcp-server/src/lib.rs@@ -0,0 +1,20 @@+//! MCP JSON-RPC 2.0 server over stdio — platform-independent core.+//!+//! Implements the Model Context Protocol (MCP) 2024-11-05 over stdio,+//! matching the interface of `libs/cua-driver` (Swift/macOS) and+//! `CuaDriver.Win` (.NET/Windows).+//!+//! # Protocol+//! - Line-delimited JSON-RPC 2.0 on stdin/stdout+//! - Methods: `initialize`, `notifications/initialized`, `tools/list`, `tools/call`+//! - Each request has `jsonrpc: "2.0"`, `id` (any), `method`, optional `params`+//! - Notifications (no `id`) are silently ignored++pub mod browser_eval;+pub mod protocol;+pub mod recording;+pub mod recording_tools;+pub mod server;+pub mod tool;++pub use recording::RecordingSession;diff --git a/libs/cua-driver-rs/crates/mcp-server/src/protocol.rs b/libs/cua-driver-rs/crates/mcp-server/src/protocol.rsnew file mode 100644index 0000000000..4dfe46adcc--- /dev/null+++ b/libs/cua-driver-rs/crates/mcp-server/src/protocol.rs@@ -0,0 +1,173 @@+//! MCP JSON-RPC 2.0 protocol types.++use serde::{Deserialize, Serialize};+use serde_json::Value;++// ── Request ──────────────────────────────────────────────────────────────────++/// An incoming JSON-RPC 2.0 request or notification.+#[derive(Debug, Deserialize)]+pub struct Request {+ #[allow(dead_code)]+ pub jsonrpc: String,+ /// Absent on notifications.+ pub id: Option<Value>,+ pub method: String,+ pub params: Option<Value>,+}++impl Request {+ pub fn is_notification(&self) -> bool {+ self.id.is_none()+ }++ pub fn tool_call(&self) -> anyhow::Result<ToolCall> {+ let params = self.params.as_ref().ok_or_else(|| anyhow::anyhow!("missing params"))?;+ let name = params+ .get("name")+ .and_then(|v| v.as_str())+ .ok_or_else(|| anyhow::anyhow!("missing tool name"))?+ .to_owned();+ let args = params+ .get("arguments")+ .cloned()+ .unwrap_or(Value::Object(Default::default()));+ Ok(ToolCall { name, args })+ }+}++pub struct ToolCall {+ pub name: String,+ pub args: Value,+}++// ── Response ─────────────────────────────────────────────────────────────────++#[derive(Debug, Serialize)]+pub struct Response {+ pub jsonrpc: &'static str,+ pub id: Value,+ #[serde(flatten)]+ pub body: ResponseBody,+}++#[derive(Debug, Serialize)]+#[serde(untagged)]+pub enum ResponseBody {+ Result { result: Value },+ Error { error: RpcError },+}++#[derive(Debug, Serialize)]+pub struct RpcError {+ pub code: i64,+ pub message: String,+}++impl Response {+ pub fn ok(id: Value, result: Value) -> Self {+ Self { jsonrpc: "2.0", id, body: ResponseBody::Result { result } }+ }++ pub fn error(id: Value, code: i64, message: impl Into<String>) -> Self {+ Self {+ jsonrpc: "2.0",+ id,+ body: ResponseBody::Error { error: RpcError { code, message: message.into() } },+ }+ }++ pub fn parse_error() -> Self {+ Self::error(Value::Null, -32700, "Parse error")+ }++ pub fn method_not_found(id: Value, method: &str) -> Self {+ Self::error(id, -32601, format!("Unknown method: {method}"))+ }+}++// ── Tool result content ───────────────────────────────────────────────────────++/// A single item in the `content` array of a `tools/call` result.+#[derive(Debug, Serialize, Clone)]+#[serde(tag = "type", rename_all = "lowercase")]+pub enum Content {+ Text {+ text: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ annotations: Option<Value>,+ },+ Image {+ data: String, // base64-encoded PNG+ #[serde(rename = "mimeType")]+ mime_type: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ annotations: Option<Value>,+ },+}++impl Content {+ pub fn text(text: impl Into<String>) -> Self {+ Content::Text { text: text.into(), annotations: None }+ }++ pub fn image_png(data_base64: String) -> Self {+ Content::Image { data: data_base64, mime_type: "image/png".into(), annotations: None }+ }++ pub fn image_jpeg(data_base64: String) -> Self {+ Content::Image { data: data_base64, mime_type: "image/jpeg".into(), annotations: None }+ }+}++/// The value placed in `result` for a `tools/call` response.+#[derive(Debug, Serialize, Default)]+pub struct ToolResult {+ pub content: Vec<Content>,+ #[serde(rename = "isError", skip_serializing_if = "Option::is_none")]+ pub is_error: Option<bool>,+ #[serde(rename = "structuredContent", skip_serializing_if = "Option::is_none")]+ pub structured_content: Option<Value>,+}++impl ToolResult {+ pub fn text(msg: impl Into<String>) -> Self {+ Self { content: vec![Content::text(msg)], ..Default::default() }+ }++ pub fn error(msg: impl Into<String>) -> Self {+ Self { content: vec![Content::text(msg)], is_error: Some(true), ..Default::default() }+ }++ pub fn with_structured(mut self, v: Value) -> Self {+ self.structured_content = Some(v);+ self+ }+}++// ── Initialize result ─────────────────────────────────────────────────────────++pub fn initialize_result() -> Value {+ serde_json::json!({+ "protocolVersion": "2025-06-18",+ "capabilities": { "tools": {} },+ "serverInfo": { "name": "cua-driver-rs", "version": env!("CARGO_PKG_VERSION") },+ "instructions": AGENT_INSTRUCTIONS+ })+}++const AGENT_INSTRUCTIONS: &str = "\+cua-driver-rs: cross-platform background computer-use automation.++Tools let you interact with any app without stealing keyboard focus or \+moving the visible cursor. Prefer element_index (AX/UIA) paths over pixel \+coordinates — they work on backgrounded/hidden windows.++Workflow per turn:+1. list_apps → confirm the target is running, get pid+2. list_windows → pick a window_id on the current Space+3. get_window_state(pid, window_id) → refresh AX/UIA snapshot, get element indices+4. click/type_text/press_key using element_index from step 3++Agent cursor: set_agent_cursor_* tools visualise where the agent is acting \+without affecting the real mouse pointer.";diff --git a/libs/cua-driver-rs/crates/mcp-server/src/recording.rs b/libs/cua-driver-rs/crates/mcp-server/src/recording.rsnew file mode 100644index 0000000000..10fdd51034--- /dev/null+++ b/libs/cua-driver-rs/crates/mcp-server/src/recording.rs@@ -0,0 +1,261 @@+//! Trajectory recording session.+//!+//! When enabled, every non-read-only, non-recording tool call writes a+//! `turn-NNNNN/action.json` file to the configured output directory.+//! When a screenshot callback is registered via `set_screenshot_fn`, it also+//! writes `screenshot.png` (extracted from `pid`/`window_id` in the args).+//!+//! Schema mirrors the Swift/Windows reference `action.json`:+//! { tool, arguments, result_summary, timestamp, t_ms_from_session_start,+//! t_start_ms_from_session_start }++use std::path::{Path, PathBuf};+use std::sync::{Mutex, OnceLock};+use std::time::{SystemTime, UNIX_EPOCH};++use serde_json::Value;++// ── Platform screenshot callback ─────────────────────────────────────────────+//+// Registered once at startup by each platform crate. Takes (window_id, pid)+// and returns raw PNG bytes, or None if capture fails. The callback is called+// synchronously from write_turn (a blocking context).++type ScreenshotFnBox = Box<dyn Fn(Option<u64>, Option<i64>) -> Option<Vec<u8>> + Send + Sync>;+static SCREENSHOT_FN: OnceLock<ScreenshotFnBox> = OnceLock::new();++/// Register the platform-specific screenshot callback. Call once at startup+/// before any tool invocations. Subsequent calls are silently ignored.+pub fn set_screenshot_fn(f: impl Fn(Option<u64>, Option<i64>) -> Option<Vec<u8>> + Send + Sync + 'static) {+ let _ = SCREENSHOT_FN.set(Box::new(f));+}++// ── Platform click-marker callback ───────────────────────────────────────────+//+// Takes (png_bytes, cx, cy) and returns modified PNG bytes with a red crosshair+// at (cx, cy), or None if drawing fails. Used to produce click.png alongside+// screenshot.png when a click-family tool is recorded.++type ClickMarkerFnBox = Box<dyn Fn(&[u8], f64, f64) -> Option<Vec<u8>> + Send + Sync>;+static CLICK_MARKER_FN: OnceLock<ClickMarkerFnBox> = OnceLock::new();++/// Register the platform-specific click-marker callback. Call once at startup.+pub fn set_click_marker_fn(f: impl Fn(&[u8], f64, f64) -> Option<Vec<u8>> + Send + Sync + 'static) {+ let _ = CLICK_MARKER_FN.set(Box::new(f));+}++/// Persistent recording session state (singleton per process).+pub struct RecordingSession {+ inner: Mutex<RecordingInner>,+}++struct RecordingInner {+ enabled: bool,+ output_dir: Option<PathBuf>,+ next_turn: u32,+ session_start_ms: u64,+ last_error: Option<String>,+}++/// Snapshot of the current recording state (cheap to clone).+#[derive(Debug, Clone)]+pub struct RecordingState {+ pub enabled: bool,+ pub output_dir: Option<String>,+ pub next_turn: u32,+ pub last_error: Option<String>,+}++impl RecordingSession {+ pub fn new() -> Self {+ Self {+ inner: Mutex::new(RecordingInner {+ enabled: false,+ output_dir: None,+ next_turn: 1,+ session_start_ms: 0,+ last_error: None,+ }),+ }+ }++ /// Enable or disable recording. `output_dir` is required when `enabled=true`.+ pub fn configure(&self, enabled: bool, output_dir: Option<&str>) -> anyhow::Result<()> {+ let mut inner = self.inner.lock().unwrap();+ if !enabled {+ inner.enabled = false;+ inner.output_dir = None;+ inner.next_turn = 1;+ inner.session_start_ms = 0;+ inner.last_error = None;+ return Ok(());+ }++ let dir_str = output_dir+ .ok_or_else(|| anyhow::anyhow!("output_dir is required when enabling recording"))?;+ let dir = expand_tilde(dir_str);+ std::fs::create_dir_all(&dir)?;++ // Write initial session.json.+ let session_payload = serde_json::json!({+ "schema_version": 1,+ "started_at_monotonic_ms": now_ms(),+ "video": { "present": false },+ "cursor": { "present": false }+ });+ let _ = write_json_atomic(+ &dir.join("session.json"),+ &session_payload,+ );++ inner.enabled = true;+ inner.output_dir = Some(dir);+ inner.next_turn = 1;+ inner.session_start_ms = now_ms();+ inner.last_error = None;+ Ok(())+ }++ /// Return a snapshot of the current state (non-blocking).+ pub fn current_state(&self) -> RecordingState {+ let inner = self.inner.lock().unwrap();+ RecordingState {+ enabled: inner.enabled,+ output_dir: inner.output_dir.as_ref().map(|p| p.to_string_lossy().into_owned()),+ next_turn: inner.next_turn,+ last_error: inner.last_error.clone(),+ }+ }++ /// Record a completed tool call. No-op when recording is disabled.+ /// `start_ms` — wall-clock ms at invocation start (use `now_ms()` before calling the tool).+ pub fn record(+ &self,+ tool_name: &str,+ args: &Value,+ result_text: &str,+ start_ms: u64,+ ) {+ let (turn_dir, session_start_ms) = {+ let mut inner = self.inner.lock().unwrap();+ if !inner.enabled {+ return;+ }+ let out = match inner.output_dir.clone() {+ Some(o) => o,+ None => return,+ };+ let idx = inner.next_turn;+ inner.next_turn += 1;+ (out.join(format!("turn-{idx:05}")), inner.session_start_ms)+ };++ if let Err(e) = write_turn(+ &turn_dir,+ tool_name,+ args,+ result_text,+ start_ms,+ session_start_ms,+ ) {+ let mut inner = self.inner.lock().unwrap();+ inner.last_error = Some(e.to_string());+ }+ }+}++impl Default for RecordingSession {+ fn default() -> Self { Self::new() }+}++// ── helpers ───────────────────────────────────────────────────────────────────++fn write_turn(+ turn_dir: &Path,+ tool_name: &str,+ args: &Value,+ result_text: &str,+ start_ms: u64,+ session_start_ms: u64,+) -> anyhow::Result<()> {+ std::fs::create_dir_all(turn_dir)?;+ let now = now_ms();++ // Extract window_id and pid from args for screenshot capture.+ let window_id = args.get("window_id").and_then(|v| v.as_u64());+ let pid = args.get("pid").and_then(|v| v.as_i64());++ // Extract click point for click-family tools.+ let click_point: Option<(f64, f64)> = if matches!(+ tool_name, "click" | "double_click" | "right_click"+ ) {+ match (args.get("x").and_then(|v| v.as_f64()), args.get("y").and_then(|v| v.as_f64())) {+ (Some(x), Some(y)) => Some((x, y)),+ _ => None,+ }+ } else {+ None+ };++ let mut payload = serde_json::json!({+ "tool": tool_name,+ "arguments": args,+ "result_summary": result_text,+ "timestamp": iso_now(),+ "t_ms_from_session_start": now.saturating_sub(session_start_ms),+ "t_start_ms_from_session_start": start_ms.saturating_sub(session_start_ms),+ });+ if let Some((cx, cy)) = click_point {+ payload["click_point"] = serde_json::json!({"x": cx, "y": cy});+ }+ write_json_atomic(&turn_dir.join("action.json"), &payload)?;++ // Capture screenshot if a callback is registered.+ if let Some(screenshot_fn) = SCREENSHOT_FN.get() {+ if let Some(png_bytes) = screenshot_fn(window_id, pid) {+ let _ = std::fs::write(turn_dir.join("screenshot.png"), &png_bytes);+ // Write click.png (screenshot + red crosshair) for click-family tools.+ if let Some((cx, cy)) = click_point {+ if let Some(marker_fn) = CLICK_MARKER_FN.get() {+ if let Some(click_png) = marker_fn(&png_bytes, cx, cy) {+ let _ = std::fs::write(turn_dir.join("click.png"), &click_png);+ }+ }+ }+ }+ }++ Ok(())+}++fn write_json_atomic(path: &Path, value: &Value) -> anyhow::Result<()> {+ let tmp = path.with_extension("tmp");+ std::fs::write(&tmp, serde_json::to_string_pretty(value)?)?;+ std::fs::rename(&tmp, path)?;+ Ok(())+}++/// Current wall-clock time as milliseconds since Unix epoch.+pub fn now_ms() -> u64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default()+ .as_millis() as u64+}++fn iso_now() -> String {+ let d = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default();+ // Format as fractional Unix seconds (simple, unambiguous, machine-readable).+ format!("{:.3}", d.as_secs_f64())+}++fn expand_tilde(path: &str) -> PathBuf {+ if let Some(rest) = path.strip_prefix("~/") {+ if let Ok(home) = std::env::var("HOME") {+ return PathBuf::from(home).join(rest);+ }+ }+ PathBuf::from(path)+}diff --git a/libs/cua-driver-rs/crates/mcp-server/src/recording_tools.rs b/libs/cua-driver-rs/crates/mcp-server/src/recording_tools.rsnew file mode 100644index 0000000000..b607e92c99--- /dev/null+++ b/libs/cua-driver-rs/crates/mcp-server/src/recording_tools.rs@@ -0,0 +1,382 @@+//! Platform-independent recording / replay tools.+//!+//! Registered on all platforms via `ToolRegistry::register_recording_tools()`.+//!+//! - `set_recording` — enable/disable trajectory recording to disk+//! - `get_recording_state` — query current recording state+//! - `replay_trajectory` — replay a previously recorded trajectory++use std::sync::{Arc, OnceLock, Weak};++use async_trait::async_trait;+use serde_json::{json, Value};++use crate::{+ protocol::ToolResult,+ recording::{RecordingSession, RecordingState},+ tool::{Tool, ToolDef, ToolRegistry},+};++// ── Process-global weak reference to the registry (for replay) ───────────────+//+// Set once by `ToolRegistry::init_replay(weak)` after `Arc::new(registry)`.++static REPLAY_REGISTRY: OnceLock<Weak<ToolRegistry>> = OnceLock::new();++/// Called from `main.rs` after wrapping the registry in `Arc`.+pub fn init_replay_registry(weak: Weak<ToolRegistry>) {+ let _ = REPLAY_REGISTRY.set(weak);+}++fn get_replay_registry() -> Option<Arc<ToolRegistry>> {+ REPLAY_REGISTRY.get()?.upgrade()+}++// ── set_recording ─────────────────────────────────────────────────────────────++pub struct SetRecordingTool {+ session: Arc<RecordingSession>,+}++impl SetRecordingTool {+ pub fn new(session: Arc<RecordingSession>) -> Self { Self { session } }+}++static SET_REC_DEF: OnceLock<ToolDef> = OnceLock::new();++#[async_trait]+impl Tool for SetRecordingTool {+ fn def(&self) -> &ToolDef {+ SET_REC_DEF.get_or_init(|| ToolDef {+ name: "set_recording".into(),+ // Description ported from Swift `SetRecordingTool.swift`.+ description: "Toggle trajectory recording. When enabled, every subsequent \+ action-tool invocation (click, right_click, scroll, type_text, press_key, \+ hotkey, set_value) writes a turn folder under `output_dir`:\n\n\+ - `app_state.json` — post-action AX/UIA snapshot for the target pid.\n\+ - `screenshot.png` — post-action per-window screenshot of the target's \+ frontmost on-screen window.\n\+ - `action.json` — tool name, full input arguments, result summary, pid, \+ click point (when applicable), ISO-8601 timestamp.\n\+ - `click.png` — for click-family actions only, `screenshot.png` with a \+ red dot drawn at the click point.\n\n\+ Turn folders are named `turn-00001/`, `turn-00002/`, etc. Turn \+ numbering restarts at 1 each time recording is (re-)enabled.\n\n\+ Required when `enabled=true`: `output_dir`. Ignored when \+ `enabled=false`.\n\n\+ State persists for the life of the daemon / MCP session; a restart \+ resets to disabled with no on-disk state.".into(),+ input_schema: json!({+ "type": "object",+ "required": ["enabled"],+ "properties": {+ "enabled": { "type": "boolean", "description": "True to start recording subsequent action tool calls; false to stop." },+ "output_dir": { "type": "string", "description": "Absolute or ~-rooted directory where turn folders are written. Required when enabled=true." },+ "video_experimental": {+ "type": "boolean",+ "description": "Experimental: capture main display to <output_dir>/recording.mp4. Off by default. Ignored when enabled=false."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ // Swift annotation: idempotentHint: true.+ idempotent: true,+ open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let enabled = match args.get("enabled").and_then(|v| v.as_bool()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required boolean field `enabled`."),+ };+ let output_dir = args.get("output_dir").and_then(|v| v.as_str());+ if enabled && output_dir.map(|s| s.is_empty()).unwrap_or(true) {+ return ToolResult::error("`output_dir` is required when enabling recording.");+ }+ let video_experimental = args.get("video_experimental").and_then(|v| v.as_bool()).unwrap_or(false);++ match self.session.configure(enabled, output_dir) {+ Ok(()) => {+ let state = self.session.current_state();+ let msg = if state.enabled {+ let video_note = if video_experimental {+ " (video_experimental is not yet implemented on this platform)"+ } else { "" };+ // Match Swift text format 1:1: `"✅ Recording enabled -> <path>"`.+ format!("✅ Recording enabled -> {}{video_note}",+ state.output_dir.as_deref().unwrap_or("?"))+ } else {+ "✅ Recording disabled.".to_owned()+ };+ ToolResult::text(msg).with_structured(recording_state_json(&state))+ }+ Err(e) => ToolResult::error(format!("Failed to enable recording: {e}")),+ }+ }+}++// ── get_recording_state ───────────────────────────────────────────────────────++pub struct GetRecordingStateTool {+ session: Arc<RecordingSession>,+}++impl GetRecordingStateTool {+ pub fn new(session: Arc<RecordingSession>) -> Self { Self { session } }+}++static GET_REC_DEF: OnceLock<ToolDef> = OnceLock::new();++#[async_trait]+impl Tool for GetRecordingStateTool {+ fn def(&self) -> &ToolDef {+ GET_REC_DEF.get_or_init(|| ToolDef {+ name: "get_recording_state".into(),+ // Description ported from Swift `GetRecordingStateTool.swift`.+ description: "Report the current trajectory recorder state: whether recording \+ is enabled, the output directory (when enabled), and the 1-based counter \+ for the next turn folder that will be written. Counter increments on every \+ recorded action tool call and resets to 1 each time recording is \+ (re-)enabled.\n\n\+ Pure read-only.".into(),+ input_schema: json!({ "type": "object", "properties": {}, "additionalProperties": false }),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+ }++ async fn invoke(&self, _args: Value) -> ToolResult {+ let state = self.session.current_state();+ // Match Swift text format 1:1:+ // "✅ recording: enabled output_dir=<path> next_turn=<N>"+ // "✅ recording: disabled"+ let summary = if state.enabled {+ format!(+ "recording: enabled output_dir={} next_turn={}",+ state.output_dir.as_deref().unwrap_or("?"),+ state.next_turn+ )+ } else {+ "recording: disabled".to_owned()+ };+ ToolResult::text(format!("✅ {summary}")).with_structured(recording_state_json(&state))+ }+}++// ── replay_trajectory ─────────────────────────────────────────────────────────++pub struct ReplayTrajectoryTool;++static REPLAY_DEF: OnceLock<ToolDef> = OnceLock::new();++#[async_trait]+impl Tool for ReplayTrajectoryTool {+ fn def(&self) -> &ToolDef {+ REPLAY_DEF.get_or_init(|| ToolDef {+ name: "replay_trajectory".into(),+ // Description ported from Swift `ReplayTrajectoryTool.swift`+ // with its caveats about element-indexed actions and recording-+ // during-replay semantics.+ description: "Replay a recorded trajectory by re-invoking every turn's tool call \+ in lexical order. `dir` must point at a directory previously written by \+ `set_recording`. Each `turn-NNNNN/` is parsed for `action.json`, and the \+ recorded tool is called with its recorded `arguments` via the same dispatch \+ path an MCP / CLI call uses.\n\n\+ Caveats:\n\+ - Element-indexed actions (`click({pid, element_index})` etc.) will fail \+ because element indices are per-snapshot and don't survive across \+ sessions. Pixel clicks (`click({pid, x, y})`) and all keyboard tools \+ replay cleanly. Failures are reported but don't stop replay unless \+ `stop_on_error` is true.\n\+ - `get_window_state` and other read-only tools are NOT currently recorded, \+ so replays do not re-populate the per-(pid, window_id) element cache.\n\+ - If recording is ENABLED while replay runs, the replay itself is recorded \+ into the currently configured output directory. That's deliberate: \+ recording a replay against a new build and diffing the two trajectories \+ is the regression-test workflow.".into(),+ input_schema: json!({+ "type": "object",+ "required": ["dir"],+ "properties": {+ "dir": { "type": "string", "description": "Trajectory directory previously written by `set_recording`. Absolute or ~-rooted." },+ "delay_ms": { "type": "integer", "minimum": 0, "maximum": 10000, "description": "Milliseconds to sleep between turns, for human-observable pacing. Default 500." },+ "stop_on_error": { "type": "boolean", "description": "Stop replay on the first tool-call error. Default true — set false to best-effort through the full trajectory." }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let dir_str = match args.get("dir").and_then(|v| v.as_str()) {+ Some(v) if !v.is_empty() => v.to_owned(),+ _ => return ToolResult::error("Missing required string field `dir`."),+ };+ let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(500).min(10_000);+ let stop_on_error = args.get("stop_on_error").and_then(|v| v.as_bool()).unwrap_or(true);++ // Expand ~/+ let dir = {+ let p = std::path::PathBuf::from(&dir_str);+ if dir_str.starts_with("~/") {+ if let Ok(home) = std::env::var("HOME") {+ std::path::PathBuf::from(home).join(&dir_str[2..])+ } else { p }+ } else { p }+ };++ if !dir.exists() {+ return ToolResult::error(format!("Trajectory directory does not exist: {}", dir.display()));+ }++ // Collect and sort turn-NNNNN directories.+ let mut turn_dirs: Vec<_> = std::fs::read_dir(&dir)+ .map(|rd| {+ rd.filter_map(|e| e.ok())+ .map(|e| e.path())+ .filter(|p| p.is_dir() && p.file_name()+ .and_then(|n| n.to_str())+ .map(|n| n.starts_with("turn-"))+ .unwrap_or(false))+ .collect::<Vec<_>>()+ })+ .unwrap_or_default();+ turn_dirs.sort();++ if turn_dirs.is_empty() {+ return ToolResult::error(format!("No turn-NNNNN folders found under {}", dir.display()));+ }++ let registry = match get_replay_registry() {+ Some(r) => r,+ None => return ToolResult::error("Replay not available: registry not initialised yet."),+ };++ let mut attempted = 0u32;+ let mut succeeded = 0u32;+ let mut failed = 0u32;+ let mut turns_json = Vec::new();+ let mut first_failure: Option<(String, String, String)> = None;++ for turn_dir in &turn_dirs {+ let turn_name = turn_dir.file_name()+ .and_then(|n| n.to_str())+ .unwrap_or("?")+ .to_owned();++ let action_path = turn_dir.join("action.json");+ let (tool_name, tool_args) = match parse_action_json(&action_path) {+ Ok(v) => v,+ Err(e) => {+ failed += 1;+ if first_failure.is_none() {+ first_failure = Some((turn_name.clone(), "action.json".into(), e.to_string()));+ }+ turns_json.push(json!({+ "turn": turn_name,+ "ok": false,+ "parse_error": e.to_string()+ }));+ if stop_on_error { break; }+ continue;+ }+ };++ attempted += 1;+ let result = registry.invoke(&tool_name, tool_args).await;+ let is_err = result.is_error.unwrap_or(false);+ let summary = result.content.iter()+ .find_map(|c| {+ if let crate::protocol::Content::Text { text, .. } = c {+ Some(text.as_str())+ } else { None }+ })+ .unwrap_or("")+ .to_owned();++ turns_json.push(json!({+ "turn": turn_name,+ "tool": &tool_name,+ "ok": !is_err,+ "result_summary": &summary,+ }));++ if is_err {+ failed += 1;+ if first_failure.is_none() {+ first_failure = Some((turn_name.clone(), tool_name.clone(), summary));+ }+ if stop_on_error { break; }+ } else {+ succeeded += 1;+ }++ if delay_ms > 0 {+ tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;+ }+ }++ let dir_name = dir.file_name()+ .and_then(|n| n.to_str())+ .unwrap_or("?");+ let mut summary_text = format!(+ "replay {dir_name}: attempted={attempted} succeeded={succeeded} failed={failed}"+ );+ if let Some((ref turn, ref tool, _)) = first_failure {+ summary_text.push_str(&format!(" first_failure={turn}:{tool}"));+ }++ let mut structured = json!({+ "directory": dir.to_string_lossy(),+ "attempted": attempted,+ "succeeded": succeeded,+ "failed": failed,+ "stop_on_error": stop_on_error,+ "turns": turns_json,+ });+ if let Some((turn, tool, error)) = first_failure {+ structured["first_failure"] = json!({ "turn": turn, "tool": tool, "error": error });+ }++ ToolResult::text(summary_text)+ .with_structured(structured)+ }+}++// ── helpers ───────────────────────────────────────────────────────────────────++fn recording_state_json(state: &RecordingState) -> Value {+ json!({+ // "recording" mirrors the Swift field name for parity.+ "recording": state.enabled,+ "enabled": state.enabled,+ "output_dir": state.output_dir,+ "next_turn": state.next_turn,+ "last_error": state.last_error,+ })+}++fn parse_action_json(path: &std::path::Path) -> anyhow::Result<(String, Value)> {+ if !path.exists() {+ anyhow::bail!("Missing action.json");+ }+ let text = std::fs::read_to_string(path)?;+ let obj: Value = serde_json::from_str(&text)?;+ let tool = obj.get("tool")+ .and_then(|v| v.as_str())+ .ok_or_else(|| anyhow::anyhow!("action.json missing 'tool' string field"))?+ .to_owned();+ let tool_args = obj.get("arguments").cloned().unwrap_or(Value::Object(Default::default()));+ Ok((tool, tool_args))+}diff --git a/libs/cua-driver-rs/crates/mcp-server/src/server.rs b/libs/cua-driver-rs/crates/mcp-server/src/server.rsnew file mode 100644index 0000000000..0bae57a952--- /dev/null+++ b/libs/cua-driver-rs/crates/mcp-server/src/server.rs@@ -0,0 +1,82 @@+//! Async MCP stdio server loop.++use std::sync::Arc;+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};+use tracing::{debug, error, warn};++use crate::protocol::{initialize_result, Request, Response};+use crate::tool::ToolRegistry;++/// Run the MCP server, reading JSON-RPC lines from stdin and writing+/// responses to stdout. Exits when stdin reaches EOF or a fatal I/O+/// error occurs.+pub async fn run(registry: Arc<ToolRegistry>) -> anyhow::Result<()> {+ let stdin = tokio::io::stdin();+ let stdout = tokio::io::stdout();+ let mut reader = BufReader::new(stdin);+ let mut writer = tokio::io::BufWriter::new(stdout);+ let mut line = String::new();++ loop {+ line.clear();+ let n = reader.read_line(&mut line).await?;+ if n == 0 {+ // EOF+ break;+ }+ let trimmed = line.trim();+ if trimmed.is_empty() {+ continue;+ }+ debug!(raw = trimmed, "→ request");++ let response = match serde_json::from_str::<Request>(trimmed) {+ Err(e) => {+ error!("JSON parse error: {e}");+ Response::parse_error()+ }+ Ok(req) if req.is_notification() => {+ // Notifications are silently dropped.+ continue;+ }+ Ok(req) => {+ let id = req.id.clone().unwrap_or(serde_json::Value::Null);+ handle_request(req, id, ®istry).await+ }+ };++ let serialized = serde_json::to_string(&response)+ .unwrap_or_else(|e| format!(r#"{{"jsonrpc":"2.0","id":null,"error":{{"code":-32603,"message":"serialize error: {e}"}}}}"#));+ debug!(raw = %serialized, "← response");++ writer.write_all(serialized.as_bytes()).await?;+ writer.write_all(b"\n").await?;+ writer.flush().await?;+ }++ Ok(())+}++async fn handle_request(req: Request, id: serde_json::Value, registry: &Arc<ToolRegistry>) -> Response {+ match req.method.as_str() {+ "initialize" => Response::ok(id, initialize_result()),++ "tools/list" => Response::ok(id, registry.tools_list()),++ "tools/call" => match req.tool_call() {+ Err(e) => Response::error(id, -32602, format!("Invalid params: {e}")),+ Ok(call) => {+ let result = registry.invoke(&call.name, call.args).await;+ match serde_json::to_value(result) {+ Ok(v) => Response::ok(id, v),+ Err(e) => Response::error(id, -32603, format!("Serialize error: {e}")),+ }+ }+ },++ other => {+ warn!(method = other, "unknown method");+ Response::method_not_found(id, other)+ }+ }+}diff --git a/libs/cua-driver-rs/crates/mcp-server/src/tool.rs b/libs/cua-driver-rs/crates/mcp-server/src/tool.rsnew file mode 100644index 0000000000..da020ab8ea--- /dev/null+++ b/libs/cua-driver-rs/crates/mcp-server/src/tool.rs@@ -0,0 +1,171 @@+//! Tool trait and registry.++use std::collections::HashMap;+use std::sync::Arc;++use async_trait::async_trait;+use serde_json::Value;++use crate::{+ browser_eval::BrowserEvalTool,+ protocol::{Content, ToolResult},+ recording::{now_ms, RecordingSession},+ recording_tools::{+ GetRecordingStateTool, ReplayTrajectoryTool, SetRecordingTool,+ init_replay_registry,+ },+};++/// Metadata for a single tool.+#[derive(Debug, Clone)]+pub struct ToolDef {+ pub name: String,+ pub description: String,+ pub input_schema: Value,+ pub read_only: bool,+ pub destructive: bool,+ pub idempotent: bool,+ pub open_world: bool,+}++impl ToolDef {+ pub fn to_list_entry(&self) -> Value {+ serde_json::json!({+ "name": self.name,+ "description": self.description,+ "inputSchema": self.input_schema,+ "annotations": {+ "readOnlyHint": self.read_only,+ "destructiveHint": self.destructive,+ "idempotentHint": self.idempotent,+ "openWorldHint": self.open_world,+ }+ })+ }+}++/// A callable tool handler. Object-safe — uses `Box<dyn Tool>`.+#[async_trait]+pub trait Tool: Send + Sync {+ fn def(&self) -> &ToolDef;+ async fn invoke(&self, args: Value) -> ToolResult;+}++/// Thread-safe collection of all registered tools.+pub struct ToolRegistry {+ tools: HashMap<String, Box<dyn Tool>>,+ /// Ordered list of tool names for `tools/list`.+ order: Vec<String>,+ /// Shared recording session — auto-records each non-read-only tool call.+ pub recording: Arc<RecordingSession>,+}++impl ToolRegistry {+ pub fn new() -> Self {+ Self {+ tools: HashMap::new(),+ order: Vec::new(),+ recording: Arc::new(RecordingSession::new()),+ }+ }++ pub fn register(&mut self, tool: Box<dyn Tool>) {+ let name = tool.def().name.clone();+ self.order.push(name.clone());+ self.tools.insert(name, tool);+ }++ /// Register the three platform-independent recording/replay tools.+ /// Call this after all platform tools have been registered.+ pub fn register_recording_tools(&mut self) {+ let session = self.recording.clone();+ self.register(Box::new(SetRecordingTool::new(session.clone())));+ self.register(Box::new(GetRecordingStateTool::new(session)));+ self.register(Box::new(ReplayTrajectoryTool));+ self.register(Box::new(BrowserEvalTool));+ }++ /// Wire up the replay tool's weak self-reference.+ /// Call this once, immediately after `Arc::new(registry)`.+ pub fn init_self_weak(self: &Arc<Self>) {+ init_replay_registry(Arc::downgrade(self));+ }++ pub fn tools_list(&self) -> Value {+ let list: Vec<Value> =+ self.order.iter().filter_map(|n| self.tools.get(n)).map(|t| t.def().to_list_entry()).collect();+ serde_json::json!({ "tools": list })+ }++ /// Iterate over (name, &ToolDef) in registration order.+ pub fn iter_defs(&self) -> impl Iterator<Item = (&str, &ToolDef)> {+ self.order.iter().filter_map(move |n| {+ self.tools.get(n).map(|t| (n.as_str(), t.def()))+ })+ }++ /// Get a tool's ToolDef by name, or None if unknown.+ pub fn get_def(&self, name: &str) -> Option<&ToolDef> {+ self.tools.get(name).map(|t| t.def())+ }++ /// List all tool names in registration order.+ pub fn tool_names(&self) -> impl Iterator<Item = &str> {+ self.order.iter().map(|s| s.as_str())+ }++ /// Invoke a tool by name and (if recording is enabled) write its result to disk.+ pub async fn invoke(&self, name: &str, args: Value) -> ToolResult {+ // Capture start time for recording timestamps.+ let start_ms = now_ms();++ // Deprecated alias: `type_text_chars` → `type_text`. Swift's+ // ToolRegistry.swift keeps the same alias (with stderr warning) for+ // backwards compatibility with hermes-agent builds that still emit+ // the old name. Aliased name is intentionally not registered, so it+ // never appears in tools/list.+ let resolved_name: &str = match name {+ "type_text_chars" => {+ eprintln!("[cua-driver-rs] deprecated tool name 'type_text_chars' — use 'type_text' instead.");+ "type_text"+ }+ other => other,+ };++ let result = match self.tools.get(resolved_name) {+ Some(tool) => tool.invoke(args.clone()).await,+ None => return ToolResult::error(format!("Unknown tool: {name}")),+ };+ // Use the original name for downstream code paths below so the+ // exit-code matching and recording paths keep treating the alias+ // as a distinct call site.+ let name = resolved_name;++ // Record non-read-only, non-recording tool calls.+ let should_record = self.tools.get(name)+ .map(|t| !t.def().read_only)+ .unwrap_or(false)+ && !matches!(+ name,+ "set_recording" | "get_recording_state" | "replay_trajectory"+ );++ if should_record {+ let result_text = result.content.iter()+ .find_map(|c| {+ if let Content::Text { text, .. } = c { Some(text.as_str()) }+ else { None }+ })+ .unwrap_or("");+ self.recording.record(name, &args, result_text, start_ms);+ }++ result+ }+}++impl Default for ToolRegistry {+ fn default() -> Self {+ Self::new()+ }+}diff --git a/libs/cua-driver-rs/crates/platform-linux/Cargo.toml b/libs/cua-driver-rs/crates/platform-linux/Cargo.tomlnew file mode 100644index 0000000000..5b00aa1227--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/Cargo.toml@@ -0,0 +1,23 @@+[package]+name = "platform-linux"+version.workspace = true+edition.workspace = true++[dependencies]+tokio = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+anyhow = { workspace = true }+thiserror = { workspace = true }+tracing = { workspace = true }+async-trait = "0.1"+mcp-server = { path = "../mcp-server" }+cursor-overlay = { path = "../cursor-overlay" }+# tiny-skia for cursor rendering (shared cross-platform)+tiny-skia = { version = "0.11", default-features = false, features = ["std"] }++[target.'cfg(target_os = "linux")'.dependencies]+# X11 background input + window enumeration+x11rb = { version = "0.13", features = ["xinput", "randr", "xfixes", "composite", "shape"] }+base64 = { workspace = true }+image = { workspace = true }diff --git a/libs/cua-driver-rs/crates/platform-linux/src/atspi/cache.rs b/libs/cua-driver-rs/crates/platform-linux/src/atspi/cache.rsnew file mode 100644index 0000000000..16df860ed1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/atspi/cache.rs@@ -0,0 +1,45 @@+//! AT-SPI element cache for Linux.+//! Stores element keys (u64 hash) indexed by (pid, xid) → element_index.++use super::AtspiNode;+use std::collections::HashMap;+use std::sync::Mutex;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct CacheKey { pub pid: u32, pub xid: u64 }++pub struct CachedSnapshot {+ /// element_index → element_key (opaque AT-SPI path hash).+ pub elements: Vec<u64>,+}++pub struct ElementCache {+ inner: Mutex<HashMap<CacheKey, CachedSnapshot>>,+}++impl ElementCache {+ pub fn new() -> Self { Self { inner: Mutex::new(HashMap::new()) } }++ pub fn update(&self, pid: u32, xid: u64, nodes: &[AtspiNode]) {+ let elements: Vec<u64> = nodes.iter()+ .filter(|n| n.element_index.is_some())+ .map(|n| n.element_key)+ .collect();+ self.inner.lock().unwrap().insert(CacheKey { pid, xid }, CachedSnapshot { elements });+ }++ pub fn get_element_key(&self, pid: u32, xid: u64, idx: usize) -> Option<u64> {+ self.inner.lock().unwrap()+ .get(&CacheKey { pid, xid })?+ .elements.get(idx).copied()+ }++ pub fn element_count(&self, pid: u32, xid: u64) -> usize {+ self.inner.lock().unwrap()+ .get(&CacheKey { pid, xid })+ .map(|s| s.elements.len())+ .unwrap_or(0)+ }+}++impl Default for ElementCache { fn default() -> Self { Self::new() } }diff --git a/libs/cua-driver-rs/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver-rs/crates/platform-linux/src/atspi/mod.rsnew file mode 100644index 0000000000..78b4feb742--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/atspi/mod.rs@@ -0,0 +1,439 @@+//! AT-SPI accessibility tree walking for Linux.+//!+//! AT-SPI2 is accessible via D-Bus. Rather than linking the full D-Bus library,+//! we query AT-SPI via the `gdbus` or `dbus-send` CLI tool as a subprocess,+//! or we use the `atspi` Rust crate if available.+//!+//! For a pure-Rust implementation without D-Bus library deps, we shell out to:+//! python3 -c "import pyatspi; ..." (if available)+//! OR+//! Use x11rb to read basic XA_WM_* properties as a fallback tree.+//!+//! The fallback produces a simplified tree with window title and role.++use anyhow::Result;+use std::process::Command;++pub mod cache;+pub use cache::ElementCache;++#[derive(Clone, Debug)]+pub struct AtspiNode {+ pub element_index: Option<usize>,+ pub role: String,+ pub name: Option<String>,+ pub value: Option<String>,+ pub description: Option<String>,+ pub actions: Vec<String>,+ /// For pyatspi path: element_key = element_index as u64.+ /// For X11 fallback: element_key = xid.+ pub element_key: u64,+}++pub struct AtspiTreeResult {+ pub tree_markdown: String,+ pub nodes: Vec<AtspiNode>,+}++/// Walk the AT-SPI tree for a window identified by (pid, xid).+/// Falls back to a minimal X11 property tree if AT-SPI is unavailable.+pub fn walk_tree(pid: u32, xid: u64, query: Option<&str>) -> AtspiTreeResult {+ // Try pyatspi bridge first (most complete).+ if let Ok((raw_md, nodes)) = walk_via_pyatspi(pid) {+ let md = if let Some(q) = query { filter_tree(&raw_md, q) } else { raw_md };+ return AtspiTreeResult { tree_markdown: md, nodes };+ }++ // Fallback: X11 window properties as minimal tree.+ walk_via_x11_properties(xid, query)+}++/// Perform the first advertised action on element `idx` within pid's app tree.+/// Returns Ok(action_name) on success.+pub fn perform_action(pid: u32, idx: usize) -> Result<String> {+ let script = format!(r#"+import pyatspi, sys++elements = []+def collect(acc):+ try:+ ai = acc.queryAction()+ if ai.nActions > 0:+ elements.append((acc, [ai.getName(i) for i in range(ai.nActions)]))+ except: pass+ for child in acc:+ collect(child)++desktop = pyatspi.Registry.getDesktop(0)+for app in desktop:+ if app.get_process_id() == {pid}:+ for win in app:+ collect(win)+ break++if {idx} < len(elements):+ elem, actions = elements[{idx}]+ elem.queryAction().doAction(0)+ print(actions[0])+else:+ print("ERROR: element {idx} not found (total: " + str(len(elements)) + ")", file=sys.stderr)+ sys.exit(1)+"#, pid = pid, idx = idx);++ let out = Command::new("python3").arg("-c").arg(&script).output()?;+ if !out.status.success() {+ anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());+ }+ Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())+}++/// Set the text value of element `idx` within pid's app tree via AT-SPI.+/// Tries `queryEditableText().setTextContents(value)` first,+/// then `queryValue().setCurrentValue(float)`.+pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {+ // Escape value for Python string literal: replace \ and ' to be safe.+ let safe_value = value.replace('\\', "\\\\").replace('\'', "\\'");+ let script = format!(r#"+import pyatspi, sys++elements = []+def collect(acc):+ try:+ ai = acc.queryAction()+ if ai.nActions > 0:+ elements.append(acc)+ except: pass+ for child in acc:+ collect(child)++desktop = pyatspi.Registry.getDesktop(0)+for app in desktop:+ if app.get_process_id() == {pid}:+ for win in app:+ collect(win)+ break++if {idx} >= len(elements):+ print(f"ERROR: element {idx} not found (total: {{len(elements)}})", file=sys.stderr)+ sys.exit(1)++elem = elements[{idx}]+try:+ et = elem.queryEditableText()+ et.setTextContents('{safe_value}')+ print("ok:text")+except:+ try:+ v = elem.queryValue()+ v.setCurrentValue(float('{safe_value}'))+ print("ok:value")+ except Exception as e:+ print(f"ERROR: {{e}}", file=sys.stderr)+ sys.exit(1)+"#, pid = pid, idx = idx, safe_value = safe_value);++ let out = Command::new("python3").arg("-c").arg(&script).output()?;+ if !out.status.success() {+ anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());+ }+ Ok(())+}++/// Get the screen-coordinate bounding box (x, y, width, height) of element `idx`.+pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> {+ let script = format!(r#"+import pyatspi, sys++elements = []+def collect(acc):+ try:+ ai = acc.queryAction()+ if ai.nActions > 0:+ elements.append(acc)+ except: pass+ for child in acc:+ collect(child)++desktop = pyatspi.Registry.getDesktop(0)+for app in desktop:+ if app.get_process_id() == {pid}:+ for win in app:+ collect(win)+ break++if {idx} >= len(elements):+ print(f"ERROR: element {idx} not found", file=sys.stderr)+ sys.exit(1)++elem = elements[{idx}]+try:+ comp = elem.queryComponent()+ ext = comp.getExtents(pyatspi.DESKTOP_COORDS)+ print(f"{{ext.x}},{{ext.y}},{{ext.width}},{{ext.height}}")+except Exception as e:+ print(f"ERROR: {{e}}", file=sys.stderr)+ sys.exit(1)+"#, pid = pid, idx = idx);++ let out = Command::new("python3").arg("-c").arg(&script).output()?;+ if !out.status.success() {+ anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());+ }+ let line = String::from_utf8_lossy(&out.stdout).trim().to_owned();+ let parts: Vec<i64> = line.split(',')+ .filter_map(|s| s.parse().ok())+ .collect();+ if parts.len() < 4 { anyhow::bail!("unexpected bounds output: {line}"); }+ Ok((parts[0] as i32, parts[1] as i32, parts[2] as u32, parts[3] as u32))+}++// ── Internal helpers ─────────────────────────────────────────────────────────++/// Walk via pyatspi subprocess. Returns (markdown, nodes) on success.+fn walk_via_pyatspi(pid: u32) -> Result<(String, Vec<AtspiNode>)> {+ let script = format!(r#"+import pyatspi, sys++def walk(acc, depth=0, idx=[0]):+ role = acc.getRoleName()+ name = acc.name or ""+ n_actions = 0+ actions = []+ try:+ ai = acc.queryAction()+ n_actions = ai.nActions+ actions = [ai.getName(i) for i in range(n_actions)]+ except:+ pass++ try:+ vobj = acc.queryValue()+ value_str = str(vobj.currentValue)+ except:+ value_str = ""++ indent = " " * depth+ if n_actions > 0:+ act_str = ','.join(actions)+ val_part = f' value="{{value_str}}"' if value_str else ''+ print(f"{{indent}}- [{{idx[0]}}] {{role}} \"{{name}}\"{{val_part}} [actions=[{{act_str}}]]")+ idx[0] += 1+ elif name:+ print(f"{{indent}}- {{role}} = \"{{name}}\"")++ for child in acc:+ walk(child, depth+1, idx)++desktop = pyatspi.Registry.getDesktop(0)+for app in desktop:+ if app.get_process_id() == {pid}:+ for win in app:+ walk(win)+ break+"#, pid = pid);++ let out = Command::new("python3")+ .arg("-c")+ .arg(&script)+ .output()?;++ if !out.status.success() || out.stdout.is_empty() {+ anyhow::bail!("pyatspi not available or returned empty");+ }++ let raw_md = String::from_utf8_lossy(&out.stdout).into_owned();+ let nodes = parse_pyatspi_nodes(&raw_md);+ Ok((raw_md, nodes))+}++/// Parse pyatspi markdown output into AtspiNode list.+///+/// Recognizes lines like:+/// ` - [3] button "OK" value="1.0" [actions=[click,press,release]]`+fn parse_pyatspi_nodes(md: &str) -> Vec<AtspiNode> {+ let mut nodes = Vec::new();+ // We need indexed_nodes in order so element_index == position in vec.+ let mut indexed: Vec<(usize, AtspiNode)> = Vec::new();++ for line in md.lines() {+ let trimmed = line.trim();+ if !trimmed.starts_with('-') { continue; }+ let rest = trimmed.trim_start_matches('-').trim();++ // Check for indexed element: `[N] role "name" ...`+ if rest.starts_with('[') {+ if let Some(close) = rest.find(']') {+ let idx_str = &rest[1..close];+ if let Ok(idx) = idx_str.parse::<usize>() {+ let after_idx = rest[close+1..].trim();++ // Parse role (first word) and name (quoted string).+ let (role, after_role) = split_first_word(after_idx);+ let (name, after_name) = parse_quoted_string(after_role.trim());++ // Parse optional value="..." field.+ let value = parse_field(after_name, "value=");++ // Parse [actions=[...]] field.+ let actions = parse_actions(after_name);++ let node = AtspiNode {+ element_index: Some(idx),+ role: role.to_owned(),+ name: if name.is_empty() { None } else { Some(name.to_owned()) },+ value: if value.is_empty() { None } else { Some(value.to_owned()) },+ description: None,+ actions,+ element_key: idx as u64,+ };+ indexed.push((idx, node));+ }+ }+ }+ }++ // Sort by index and flatten.+ indexed.sort_by_key(|(i, _)| *i);+ nodes.extend(indexed.into_iter().map(|(_, n)| n));+ nodes+}++fn split_first_word(s: &str) -> (&str, &str) {+ let s = s.trim();+ if let Some(pos) = s.find(|c: char| c.is_whitespace()) {+ (&s[..pos], &s[pos..])+ } else {+ (s, "")+ }+}++/// Parse a `"quoted string"` from the start of `s`. Returns (content, rest).+fn parse_quoted_string(s: &str) -> (&str, &str) {+ let s = s.trim();+ if !s.starts_with('"') { return ("", s); }+ let inner = &s[1..];+ if let Some(end) = inner.find('"') {+ (&inner[..end], &inner[end+1..])+ } else {+ ("", s)+ }+}++/// Parse a field like `value="something"` from a string.+fn parse_field<'a>(s: &'a str, prefix: &str) -> &'a str {+ if let Some(pos) = s.find(prefix) {+ let after = &s[pos + prefix.len()..];+ let (content, _) = parse_quoted_string(after);+ content+ } else {+ ""+ }+}++/// Parse `[actions=[click,press,release]]` from a string.+fn parse_actions(s: &str) -> Vec<String> {+ // Find [actions=[...]]+ if let Some(start) = s.find("[actions=[") {+ let inner = &s[start + "[actions=[".len()..];+ if let Some(end) = inner.find("]]") {+ return inner[..end].split(',')+ .map(|a| a.trim().to_owned())+ .filter(|a| !a.is_empty())+ .collect();+ }+ }+ vec![]+}++/// Minimal X11 property-based tree (fallback when AT-SPI is unavailable).+fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {+ use x11rb::connection::Connection;+ use x11rb::protocol::xproto::*;+ use x11rb::rust_connection::RustConnection;++ let (conn, _) = match RustConnection::connect(None) {+ Ok(r) => r,+ Err(_) => return AtspiTreeResult { tree_markdown: String::new(), nodes: vec![] },+ };++ let window = xid as u32;++ // Read window title.+ let title = get_x11_title(&conn, window).unwrap_or_default();++ // Read WM_CLASS.+ let wm_class = get_x11_wm_class(&conn, window).unwrap_or_default();++ let mut md = String::new();+ let mut nodes = vec![];++ let root_node = AtspiNode {+ element_index: Some(0),+ role: "window".into(),+ name: if title.is_empty() { None } else { Some(title.clone()) },+ value: None,+ description: if wm_class.is_empty() { None } else { Some(wm_class.clone()) },+ actions: vec!["activate".into()],+ element_key: xid,+ };+ md.push_str(&format!("- [0] window \"{}\" [actions=[activate]]\n", title));+ nodes.push(root_node);++ let raw_md = md;+ let tree_markdown = if let Some(q) = query {+ filter_tree(&raw_md, q)+ } else {+ raw_md+ };++ AtspiTreeResult { tree_markdown, nodes }+}++fn get_x11_title(conn: &x11rb::rust_connection::RustConnection, window: u32) -> Option<String> {+ use x11rb::protocol::xproto::*;+ // Try _NET_WM_NAME first.+ let net_wm_name = conn.intern_atom(false, b"_NET_WM_NAME").ok()?.reply().ok()?.atom;+ let utf8_string = conn.intern_atom(false, b"UTF8_STRING").ok()?.reply().ok()?.atom;+ if let Ok(reply) = conn.get_property(false, window, net_wm_name, utf8_string, 0, 1024).ok()?.reply() {+ if !reply.value.is_empty() {+ return Some(String::from_utf8_lossy(&reply.value).into_owned());+ }+ }+ let reply = conn.get_property(false, window, AtomEnum::WM_NAME, AtomEnum::STRING, 0, 1024).ok()?.reply().ok()?;+ Some(String::from_utf8_lossy(&reply.value).into_owned())+}++fn get_x11_wm_class(conn: &x11rb::rust_connection::RustConnection, window: u32) -> Option<String> {+ use x11rb::protocol::xproto::*;+ let reply = conn.get_property(false, window, AtomEnum::WM_CLASS, AtomEnum::STRING, 0, 512).ok()?.reply().ok()?;+ let s = String::from_utf8_lossy(&reply.value);+ // WM_CLASS is two NUL-separated strings: instance_name\0class_name\0+ Some(s.trim_end_matches('\0').replace('\0', "."))+}++fn filter_tree(markdown: &str, query: &str) -> String {+ let needle = query.to_lowercase();+ let lines: Vec<&str> = markdown.lines().collect();+ let mut ancestors: Vec<&str> = Vec::new();+ let mut last_emitted: Vec<Option<&str>> = Vec::new();+ let mut output: Vec<&str> = Vec::new();++ for line in &lines {+ let depth = line.chars().take_while(|c| *c == ' ').count() / 2;+ while ancestors.len() <= depth { ancestors.push(""); last_emitted.push(None); }+ for d in (depth+1)..ancestors.len() { last_emitted[d] = None; }+ ancestors[depth] = line;+ if line.to_lowercase().contains(&needle) {+ for d in 0..depth {+ if ancestors[d].is_empty() { continue; }+ if last_emitted[d] == Some(ancestors[d]) { continue; }+ last_emitted[d] = Some(ancestors[d]);+ output.push(ancestors[d]);+ }+ last_emitted[depth] = Some(line);+ output.push(line);+ }+ }+ if output.is_empty() { return String::new(); }+ let mut r = output.join("\n"); r.push('\n'); r+}diff --git a/libs/cua-driver-rs/crates/platform-linux/src/capture.rs b/libs/cua-driver-rs/crates/platform-linux/src/capture.rsnew file mode 100644index 0000000000..279eaab2ef--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/capture.rs@@ -0,0 +1,277 @@+//! Window screenshot on Linux.+//!+//! Strategy (in order of preference):+//! 1. `xwd -id <xid> -silent | xwdtopnm | pnmtopng` (X11, no focus change)+//! 2. `import -window <xid> png:-` (ImageMagick, widely available)+//! 3. `scrot -u <file>` (focused window fallback)+//! 4. XGetImage via x11rb (pure Rust, no subprocess)++use anyhow::{bail, Result};+use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};+use std::process::Command;++/// Capture a window by X11 XID. Returns raw PNG bytes.+pub fn screenshot_window_bytes(xid: u64) -> Result<Vec<u8>> {+ // Try `import -window <xid> png:-` (ImageMagick).+ if let Ok(bytes) = capture_via_import(xid) {+ return Ok(bytes);+ }+ // Fallback: x11rb XGetImage → returns (b64, w, h); decode the b64 back.+ let (b64, _, _) = capture_via_xgetimage(xid)?;+ use base64::Engine as _;+ let bytes = base64::engine::general_purpose::STANDARD.decode(&b64)?;+ Ok(bytes)+}++/// Capture a window by X11 XID. Returns (base64_png, width, height).+pub fn screenshot_window(xid: u64) -> Result<(String, u32, u32)> {+ // Try `import -window <xid> png:-` (ImageMagick).+ if let Ok(bytes) = capture_via_import(xid) {+ let (w, h) = png_dimensions(&bytes)?;+ return Ok((BASE64.encode(&bytes), w, h));+ }++ // Fallback: x11rb XGetImage.+ capture_via_xgetimage(xid)+}++fn capture_via_import(xid: u64) -> Result<Vec<u8>> {+ let out = Command::new("import")+ .args(["-window", &xid.to_string(), "png:-"])+ .output()?;+ if !out.status.success() || out.stdout.is_empty() {+ bail!("import failed");+ }+ Ok(out.stdout)+}++fn capture_via_xgetimage(xid: u64) -> Result<(String, u32, u32)> {+ use x11rb::connection::Connection;+ use x11rb::protocol::xproto::*;+ use x11rb::rust_connection::RustConnection;++ let (conn, _) = RustConnection::connect(None)?;+ let window = xid as u32;++ let geom = conn.get_geometry(window)?.reply()?;+ let w = geom.width as u32;+ let h = geom.height as u32;++ let img = conn.get_image(+ ImageFormat::Z_PIXMAP,+ window,+ 0, 0,+ w as u16, h as u16,+ !0u32,+ )?.reply()?;++ // The raw data is BGRA or BGRX depending on depth.+ // Encode as a minimal PNG.+ let bytes = img.data;+ let (bpp, has_alpha) = match img.depth {+ 32 => (4usize, true),+ 24 => (4usize, false),+ _ => bail!("Unsupported depth: {}", img.depth),+ };++ // Convert to RGBA.+ let mut rgba = Vec::with_capacity((w * h * 4) as usize);+ for chunk in bytes.chunks_exact(bpp) {+ let (b, g, r) = (chunk[0], chunk[1], chunk[2]);+ let a = if has_alpha { chunk[3] } else { 255 };+ rgba.extend_from_slice(&[r, g, b, a]);+ }++ let png = write_uncompressed_png(&rgba, w, h)?;+ Ok((BASE64.encode(&png), w, h))+}++/// Public version of png_dimensions for use in tool code.+pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> {+ png_dimensions(data)+}++fn png_dimensions(data: &[u8]) -> Result<(u32, u32)> {+ if data.len() < 24 { bail!("PNG too small"); }+ // Signature (8) + IHDR length (4) + "IHDR" (4) + width (4) + height (4)+ let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]);+ let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]);+ Ok((w, h))+}++// Inline minimal uncompressed PNG writer (same as platform-windows/capture.rs).+fn write_uncompressed_png(rgba: &[u8], w: u32, h: u32) -> Result<Vec<u8>> {+ let mut out = Vec::with_capacity(rgba.len() + 4096);+ out.extend_from_slice(b"\x89PNG\r\n\x1a\n");+ let ihdr: [u8; 13] = [+ (w >> 24) as u8, (w >> 16) as u8, (w >> 8) as u8, w as u8,+ (h >> 24) as u8, (h >> 16) as u8, (h >> 8) as u8, h as u8,+ 8, 6, 0, 0, 0,+ ];+ write_png_chunk(&mut out, b"IHDR", &ihdr);+ let row_bytes = (w * 4) as usize;+ let mut raw = Vec::with_capacity((row_bytes + 1) * h as usize);+ for row in 0..h as usize {+ raw.push(0u8);+ raw.extend_from_slice(&rgba[row * row_bytes..(row + 1) * row_bytes]);+ }+ let zlib_data = zlib_store(&raw);+ write_png_chunk(&mut out, b"IDAT", &zlib_data);+ write_png_chunk(&mut out, b"IEND", &[]);+ Ok(out)+}++fn write_png_chunk(out: &mut Vec<u8>, name: &[u8; 4], data: &[u8]) {+ out.extend_from_slice(&(data.len() as u32).to_be_bytes());+ out.extend_from_slice(name);+ out.extend_from_slice(data);+ out.extend_from_slice(&crc32_ieee(name, data).to_be_bytes());+}++fn zlib_store(data: &[u8]) -> Vec<u8> {+ let adler = adler32(data);+ let mut out = vec![0x78, 0x01];+ let mut pos = 0;+ loop {+ let end = (pos + 65535).min(data.len());+ let is_last = end == data.len();+ let blen = (end - pos) as u16;+ out.push(if is_last { 1 } else { 0 });+ out.extend_from_slice(&blen.to_le_bytes());+ out.extend_from_slice(&(!blen).to_le_bytes());+ out.extend_from_slice(&data[pos..end]);+ pos = end;+ if pos >= data.len() { break; }+ }+ out.extend_from_slice(&adler.to_be_bytes());+ out+}++fn adler32(data: &[u8]) -> u32 {+ let (mut s1, mut s2) = (1u32, 0u32);+ for &b in data { s1 = (s1 + b as u32) % 65521; s2 = (s2 + s1) % 65521; }+ (s2 << 16) | s1+}++/// Capture the primary display (root window) as raw PNG bytes.+pub fn screenshot_display_bytes() -> Result<Vec<u8>> {+ // Try `import -window root png:-` (ImageMagick).+ let out = Command::new("import")+ .args(["-window", "root", "png:-"])+ .output();+ if let Ok(o) = out {+ if o.status.success() && !o.stdout.is_empty() {+ return Ok(o.stdout);+ }+ }+ // Fallback: x11rb XGetImage on the root window.+ use x11rb::connection::Connection;+ use x11rb::protocol::xproto::*;+ use x11rb::rust_connection::RustConnection;+ let (conn, screen_num) = RustConnection::connect(None)?;+ let root = conn.setup().roots[screen_num].root;+ // Get root geometry.+ let geom = conn.get_geometry(root)?.reply()?;+ let w = geom.width as u32;+ let h = geom.height as u32;+ let img = conn.get_image(ImageFormat::Z_PIXMAP, root, 0, 0, w as u16, h as u16, !0u32)?.reply()?;+ let bytes = img.data;+ let bpp = match img.depth { 32 | 24 => 4usize, _ => anyhow::bail!("Unsupported depth") };+ let mut rgba = Vec::with_capacity((w * h * 4) as usize);+ for chunk in bytes.chunks_exact(bpp) {+ let (b, g, r) = (chunk[0], chunk[1], chunk[2]);+ rgba.extend_from_slice(&[r, g, b, 255]);+ }+ write_uncompressed_png(&rgba, w, h)+}++/// Capture the primary display, returning (base64_png, width, height).+pub fn screenshot_display() -> Result<(String, u32, u32)> {+ let png_bytes = screenshot_display_bytes()?;+ let (w, h) = png_dimensions(&png_bytes)?;+ Ok((BASE64.encode(&png_bytes), w, h))+}++/// Convert PNG bytes to JPEG at the given quality (1–95).+pub fn png_bytes_to_jpeg(png_bytes: &[u8], quality: u8) -> Result<Vec<u8>> {+ let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?;+ let mut buf = Vec::new();+ {+ let mut cursor = std::io::Cursor::new(&mut buf);+ let encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut cursor, quality);+ img.write_with_encoder(encoder)?;+ }+ Ok(buf)+}++/// Downscale `png_bytes` so neither dimension exceeds `max_dim`.+/// If `max_dim == 0` or the image already fits, returns a copy of the original bytes unchanged.+pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> Result<Vec<u8>> {+ if max_dim == 0 {+ return Ok(png_bytes.to_vec());+ }+ let (w, h) = png_dimensions_pub(png_bytes)?;+ if w <= max_dim && h <= max_dim {+ return Ok(png_bytes.to_vec());+ }+ let scale = max_dim as f64 / w.max(h) as f64;+ let new_w = (w as f64 * scale).round() as u32;+ let new_h = (h as f64 * scale).round() as u32;+ let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?;+ let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3);+ let mut out = Vec::new();+ resized.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?;+ Ok(out)+}++/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return modified PNG bytes.+/// Used by recording's click-marker callback to produce click.png.+pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result<Vec<u8>> {+ let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?;+ let (w, h) = (img.width(), img.height());+ let mut img = img.to_rgba8();++ let arm_len = (w as f64 / 40.0).max(12.0) as i32;+ let line_w = ((w as f64 / 400.0).max(1.5)) as i32;+ let red = image::Rgba([255u8, 26, 26, 242]);+ let cx = cx as i32;+ let cy = cy as i32;++ for lw in 0..=line_w {+ let off = lw - line_w / 2;+ for dx in -arm_len..=arm_len {+ if let Some(p) = img.get_pixel_mut_checked(+ (cx + dx).clamp(0, w as i32 - 1) as u32,+ (cy + off).clamp(0, h as i32 - 1) as u32,+ ) { *p = red; }+ }+ for dy in -arm_len..=arm_len {+ if let Some(p) = img.get_pixel_mut_checked(+ (cx + off).clamp(0, w as i32 - 1) as u32,+ (cy + dy).clamp(0, h as i32 - 1) as u32,+ ) { *p = red; }+ }+ }++ let mut out = Vec::new();+ image::DynamicImage::ImageRgba8(img)+ .write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?;+ Ok(out)+}++fn crc32_ieee(name: &[u8], data: &[u8]) -> u32 {+ const T: [u32; 256] = {+ let mut t = [0u32; 256];+ let mut i = 0usize;+ while i < 256 {+ let mut c = i as u32;+ let mut j = 0;+ while j < 8 { c = if c & 1 != 0 { 0xEDB88320 ^ (c >> 1) } else { c >> 1 }; j += 1; }+ t[i] = c; i += 1;+ }+ t+ };+ let mut crc = !0u32;+ for &b in name.iter().chain(data.iter()) { crc = T[((crc ^ b as u32) & 0xFF) as usize] ^ (crc >> 8); }+ !crc+}diff --git a/libs/cua-driver-rs/crates/platform-linux/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-linux/src/input/mod.rsnew file mode 100644index 0000000000..fea1287bc4--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/input/mod.rs@@ -0,0 +1,348 @@+//! Background input injection for Linux via X11 XSendEvent.+//!+//! XSendEvent sends synthetic events directly to a window without changing+//! input focus. This is the Linux equivalent of PostMessage on Windows.+//!+//! Note: Some apps check the `send_event` flag and ignore synthetic events+//! (e.g., some games, some security-sensitive apps). For those, the XTest+//! extension (XTestFakeKeyEvent) is the alternative, but it DOES send to+//! the focused window.++use anyhow::Result;+use std::thread::sleep;+use std::time::Duration;+use x11rb::connection::Connection;+use x11rb::protocol::xproto::*;+use x11rb::rust_connection::RustConnection;++const CLICK_DELAY_MS: u64 = 35;+const KEY_DELAY_MS: u64 = 10;++/// Send a button click (down + up) to a window at window-local coordinates.+pub fn send_click(xid: u64, x: i32, y: i32, count: usize, button: u8) -> Result<()> {+ let (conn, _) = RustConnection::connect(None)?;+ let window = xid as u32;++ // Get the root window for the display.+ let root = conn.setup().roots[0].root;++ for _ in 0..count {+ let press = ButtonPressEvent {+ response_type: BUTTON_PRESS_EVENT,+ detail: button,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root,+ event: window,+ child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: x as i16,+ event_y: y as i16,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };++ let release = ButtonReleaseEvent {+ response_type: BUTTON_RELEASE_EVENT,+ detail: button,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root,+ event: window,+ child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: x as i16,+ event_y: y as i16,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };++ conn.send_event(false, window, EventMask::BUTTON_PRESS, &press)?;+ sleep(Duration::from_millis(CLICK_DELAY_MS));+ conn.send_event(false, window, EventMask::BUTTON_RELEASE, &release)?;+ conn.flush()?;++ if count > 1 {+ sleep(Duration::from_millis(80));+ }+ }+ Ok(())+}++/// Send a press-drag-release gesture via XSendEvent (ButtonPress + MotionNotify steps + ButtonRelease).+///+/// `xid` — target window XID. `from_x/y`, `to_x/y` — window-local coords.+/// `duration_ms` — total budget. `steps` — interpolated MotionNotify events.+/// `button` — X11 button number (1=left, 2=middle, 3=right).+pub fn send_drag(+ xid: u64,+ from_x: i32,+ from_y: i32,+ to_x: i32,+ to_y: i32,+ duration_ms: u64,+ steps: usize,+ button: u8,+) -> Result<()> {+ let (conn, _) = RustConnection::connect(None)?;+ let window = xid as u32;+ let root = conn.setup().roots[0].root;+ let steps = steps.max(1);+ let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };++ // ButtonPress at start.+ let press = ButtonPressEvent {+ response_type: BUTTON_PRESS_EVENT,+ detail: button,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root, event: window, child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: from_x as i16, event_y: from_y as i16,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };+ conn.send_event(false, window, EventMask::BUTTON_PRESS, &press)?;+ conn.flush()?;+ sleep(Duration::from_millis(CLICK_DELAY_MS));++ // Interpolated MotionNotify steps.+ for i in 1..=steps {+ let t = i as f64 / steps as f64;+ let ix = from_x + ((to_x - from_x) as f64 * t).round() as i32;+ let iy = from_y + ((to_y - from_y) as f64 * t).round() as i32;+ let motion = MotionNotifyEvent {+ response_type: MOTION_NOTIFY_EVENT,+ detail: Motion::NORMAL,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root, event: window, child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: ix as i16, event_y: iy as i16,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };+ conn.send_event(false, window, EventMask::POINTER_MOTION, &motion)?;+ conn.flush()?;+ if step_delay_ms > 0 {+ sleep(Duration::from_millis(step_delay_ms));+ }+ }++ // ButtonRelease at end.+ let release = ButtonReleaseEvent {+ response_type: BUTTON_RELEASE_EVENT,+ detail: button,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root, event: window, child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: to_x as i16, event_y: to_y as i16,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };+ conn.send_event(false, window, EventMask::BUTTON_RELEASE, &release)?;+ conn.flush()?;+ Ok(())+}++/// Type a string by sending KeyPress/KeyRelease events for each character.+pub fn send_type_text(xid: u64, text: &str) -> Result<()> {+ let (conn, _) = RustConnection::connect(None)?;+ let window = xid as u32;+ let root = conn.setup().roots[0].root;++ for ch in text.chars() {+ let keycode = char_to_keycode(&conn, ch).unwrap_or(0);+ if keycode == 0 { continue; }++ let press = KeyPressEvent {+ response_type: KEY_PRESS_EVENT,+ detail: keycode,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root,+ event: window,+ child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: 0, event_y: 0,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };++ let release = KeyReleaseEvent {+ response_type: KEY_RELEASE_EVENT,+ detail: keycode,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root,+ event: window,+ child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: 0, event_y: 0,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };++ conn.send_event(false, window, EventMask::KEY_PRESS, &press)?;+ sleep(Duration::from_millis(KEY_DELAY_MS));+ conn.send_event(false, window, EventMask::KEY_RELEASE, &release)?;+ conn.flush()?;+ }+ Ok(())+}++/// Type a string with an additional `inter_char_ms` delay between each character.+pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Result<()> {+ let (conn, _) = RustConnection::connect(None)?;+ let window = xid as u32;+ let root = conn.setup().roots[0].root;++ for ch in text.chars() {+ let keycode = char_to_keycode(&conn, ch).unwrap_or(0);+ if keycode == 0 { continue; }++ let press = KeyPressEvent {+ response_type: KEY_PRESS_EVENT,+ detail: keycode,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root, event: window, child: x11rb::NONE,+ root_x: 0, root_y: 0, event_x: 0, event_y: 0,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };+ let release = KeyReleaseEvent {+ response_type: KEY_RELEASE_EVENT,+ detail: keycode,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root, event: window, child: x11rb::NONE,+ root_x: 0, root_y: 0, event_x: 0, event_y: 0,+ state: KeyButMask::from(0u16),+ same_screen: true,+ };++ conn.send_event(false, window, EventMask::KEY_PRESS, &press)?;+ sleep(Duration::from_millis(KEY_DELAY_MS));+ conn.send_event(false, window, EventMask::KEY_RELEASE, &release)?;+ conn.flush()?;+ if inter_char_ms > 0 {+ sleep(Duration::from_millis(inter_char_ms));+ }+ }+ Ok(())+}++/// Send a named key press to a window.+pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {+ let (conn, _) = RustConnection::connect(None)?;+ let window = xid as u32;+ let root = conn.setup().roots[0].root;++ let keycode = key_name_to_keycode(&conn, key)?;+ let state = modifiers_to_state(modifiers);++ let press = KeyPressEvent {+ response_type: KEY_PRESS_EVENT,+ detail: keycode,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root,+ event: window,+ child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: 0, event_y: 0,+ state,+ same_screen: true,+ };++ let release = KeyReleaseEvent {+ response_type: KEY_RELEASE_EVENT,+ detail: keycode,+ sequence: 0,+ time: x11rb::CURRENT_TIME,+ root,+ event: window,+ child: x11rb::NONE,+ root_x: 0, root_y: 0,+ event_x: 0, event_y: 0,+ state,+ same_screen: true,+ };++ conn.send_event(false, window, EventMask::KEY_PRESS, &press)?;+ sleep(Duration::from_millis(KEY_DELAY_MS));+ conn.send_event(false, window, EventMask::KEY_RELEASE, &release)?;+ conn.flush()?;+ Ok(())+}++fn char_to_keycode(conn: &RustConnection, ch: char) -> Option<u8> {+ // Use XStringToKeysym equivalent: look up by character keysym.+ // Keysym for ASCII is just the ASCII code.+ let keysym: u32 = ch as u32;+ conn.get_keyboard_mapping(8, 248).ok()?+ .reply().ok()+ .map(|km| {+ let keysyms_per = km.keysyms_per_keycode as usize;+ for (i, syms) in km.keysyms.chunks(keysyms_per).enumerate() {+ if syms.iter().any(|&s| s == keysym) {+ return Some((8 + i) as u8);+ }+ }+ None+ })+ .flatten()+}++fn key_name_to_keycode(conn: &RustConnection, key: &str) -> Result<u8> {+ // Common X11 keysym names.+ let keysym: u32 = match key.to_lowercase().as_str() {+ "return" | "enter" => 0xFF0D,+ "tab" => 0xFF09,+ "escape" | "esc" => 0xFF1B,+ "space" | " " => 0x0020,+ "backspace" => 0xFF08,+ "delete" | "del" => 0xFFFF,+ "insert" | "ins" => 0xFF63,+ "home" => 0xFF50,+ "end" => 0xFF57,+ "pageup" | "pgup" => 0xFF55,+ "pagedown" | "pgdn" => 0xFF56,+ "up" => 0xFF52,+ "down" => 0xFF54,+ "left" => 0xFF51,+ "right" => 0xFF53,+ "f1" => 0xFFBE, "f2" => 0xFFBF, "f3" => 0xFFC0, "f4" => 0xFFC1,+ "f5" => 0xFFC2, "f6" => 0xFFC3, "f7" => 0xFFC4, "f8" => 0xFFC5,+ "f9" => 0xFFC6, "f10" => 0xFFC7, "f11" => 0xFFC8, "f12" => 0xFFC9,+ "shift" => 0xFFE1, "ctrl" | "control" => 0xFFE3, "alt" => 0xFFE9, "super" | "meta" | "win" => 0xFFEB,+ "capslock" => 0xFFE5, "numlock" => 0xFF7F,+ s if s.len() == 1 => s.chars().next().unwrap() as u32,+ _ => anyhow::bail!("Unknown key: {key}"),+ };++ let km = conn.get_keyboard_mapping(8, 248)?.reply()?;+ let kpc = km.keysyms_per_keycode as usize;+ for (i, syms) in km.keysyms.chunks(kpc).enumerate() {+ if syms.iter().any(|&s| s == keysym) {+ return Ok((8 + i) as u8);+ }+ }+ anyhow::bail!("Keysym 0x{keysym:X} not in keyboard map for key '{key}'")+}++fn modifiers_to_state(modifiers: &[&str]) -> KeyButMask {+ let mut state = 0u16;+ for m in modifiers {+ match m.to_lowercase().as_str() {+ "shift" => state |= u16::from(KeyButMask::SHIFT),+ "ctrl" | "control" => state |= u16::from(KeyButMask::CONTROL),+ "alt" | "mod1" => state |= u16::from(KeyButMask::MOD1),+ "super" | "mod4" | "win" | "meta" => state |= u16::from(KeyButMask::MOD4),+ _ => {}+ }+ }+ KeyButMask::from(state)+}diff --git a/libs/cua-driver-rs/crates/platform-linux/src/lib.rs b/libs/cua-driver-rs/crates/platform-linux/src/lib.rsnew file mode 100644index 0000000000..3fb693590c--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/lib.rs@@ -0,0 +1,43 @@+//! Linux platform backend for cua-driver-rs.+//!+//! Background automation on Linux via:+//! - /proc filesystem for process enumeration+//! - X11 _NET_CLIENT_LIST for window enumeration+//! - X11 XSendEvent for background mouse/keyboard injection (no focus steal)+//! - AT-SPI D-Bus protocol for accessibility tree (via subprocess bridge)+//! - xwd/scrot/import for window screenshots+//!+//! Wayland: falls back to xdg-output and virtual-keyboard protocols when+//! running under a Wayland compositor with XWayland support.++use mcp_server::tool::ToolRegistry;++pub mod tools;+pub mod overlay;++#[cfg(target_os = "linux")]+pub mod x11;++#[cfg(target_os = "linux")]+pub mod input;++#[cfg(target_os = "linux")]+pub mod proc_fs;++#[cfg(target_os = "linux")]+pub mod capture;++#[cfg(target_os = "linux")]+pub mod atspi;++pub fn register_tools() -> ToolRegistry {+ tools::build_registry()+}++pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig) -> ToolRegistry {+ if cfg.enabled {+ overlay::init(cfg.clone());+ overlay::run_on_thread();+ }+ tools::build_registry()+}diff --git a/libs/cua-driver-rs/crates/platform-linux/src/overlay.rs b/libs/cua-driver-rs/crates/platform-linux/src/overlay.rsnew file mode 100644index 0000000000..51348f1717--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/overlay.rs@@ -0,0 +1,597 @@+//! Linux agent-cursor overlay — X11 RGBA override-redirect window.+//!+//! Architecture:+//! - Creates an override-redirect (non-reparented) X11 window with 32-bit ARGB visual+//! from XComposite. The window covers the full display area.+//! - A background thread renders frames at ~60 Hz using tiny-skia and XShmPutImage+//! (or XPutImage fallback) with XRender ARGB compositing.+//! - Mouse events pass through via `XShapeSelectInput(ShapeInput, empty-region)`.+//! - Z-ordering: `XRaiseWindow` every 80ms to stay above normal windows.+//! - Wayland: when WAYLAND_DISPLAY is set but DISPLAY is also available (XWayland),+//! the X11 path is used. Pure Wayland support is a TODO.++use std::sync::{Mutex, OnceLock};+use std::time::{Duration, Instant};++use cursor_overlay::{+ CursorConfig, CursorShape, MotionConfig, OverlayCommand, Palette, PathPlanner, PathState,+ PlannedPath,+};++// ── Global channel ────────────────────────────────────────────────────────++static CMD_TX: OnceLock<std::sync::mpsc::SyncSender<OverlayCommand>> = OnceLock::new();+static CMD_RX_CELL: Mutex<Option<std::sync::mpsc::Receiver<OverlayCommand>>> = Mutex::new(None);+static RENDER: Mutex<Option<RenderState>> = Mutex::new(None);++pub fn init(cfg: CursorConfig) {+ let (tx, rx) = std::sync::mpsc::sync_channel(4096);+ let _ = CMD_TX.set(tx);+ *CMD_RX_CELL.lock().unwrap() = Some(rx);+ *RENDER.lock().unwrap() = Some(RenderState::new(cfg));+}++pub fn send_command(cmd: OverlayCommand) {+ if let Some(tx) = CMD_TX.get() {+ let _ = tx.try_send(cmd);+ }+}++/// Spawn the overlay on a dedicated thread. Non-blocking.+pub fn run_on_thread() {+ let rx = match CMD_RX_CELL.lock().unwrap().take() {+ Some(r) => r,+ None => return,+ };++ let cfg = {+ let guard = RENDER.lock().unwrap();+ match &*guard {+ Some(rs) => rs.cfg.clone(),+ None => return,+ }+ };++ if !cfg.enabled {+ return;+ }++ std::thread::Builder::new()+ .name("cua-overlay-x11".into())+ .spawn(move || {+ run_overlay_thread(cfg, rx);+ })+ .expect("spawn overlay thread");+}++// ── Animation state ───────────────────────────────────────────────────────++struct RenderState {+ cfg: CursorConfig,+ palette: Palette,+ motion: MotionConfig,+ pos: (f64, f64),+ heading: f64,+ path: Option<PlannedPath>,+ dist: f64,+ start_t: Instant,+ spring: Option<Spring>,+ spring_tgt: Option<(f64, f64, f64)>,+ click_t: Option<f64>,+ shape: Option<CursorShape>,+ visible: bool,+ idle_secs: f64,+ idle_alpha: f64,+ pinned_wid: Option<u64>,+ scr_w: u32,+ scr_h: u32,+}++#[derive(Clone, Copy)]+struct Spring { ox: f64, oy: f64, vx: f64, vy: f64 }++impl RenderState {+ fn new(cfg: CursorConfig) -> Self {+ let palette = cfg.palette();+ let motion = cfg.motion.clone();+ let shape = cfg.shape.clone();+ RenderState {+ cfg, palette, motion, shape,+ pos: (-200.0, -200.0),+ heading: std::f64::consts::FRAC_PI_4,+ path: None,+ dist: 0.0,+ start_t: Instant::now(),+ spring: None,+ spring_tgt: None,+ click_t: None,+ visible: true,+ idle_secs: 0.0,+ idle_alpha: 1.0,+ pinned_wid: None,+ scr_w: 1920,+ scr_h: 1080,+ }+ }++ fn tick(&mut self, dt: f64) {+ let spring_k = self.motion.spring * 400.0;+ let spring_c = self.motion.spring * 20.0;++ if let Some(ref p) = self.path {+ let path_frac = (self.dist / p.length.max(1.0)).clamp(0.0, 1.0);+ let profile = 16.0 * path_frac * path_frac * (1.0 - path_frac) * (1.0 - path_frac);+ let floor = if path_frac < 0.5 { self.motion.min_start_speed } else { self.motion.min_end_speed };+ let speed = (floor + (self.motion.peak_speed - floor) * profile).max(floor);+ self.dist += speed * dt;++ let path_len = p.length.max(1.0);+ if self.dist >= path_len {+ let end = p.sample(path_len);+ let end_heading = p.end_visual_heading;+ let vh = end.heading;+ self.spring = Some(Spring {+ ox: 0.0, oy: 0.0,+ vx: speed * 0.5 * vh.cos(),+ vy: speed * 0.5 * vh.sin(),+ });+ self.spring_tgt = Some((end.x, end.y, end_heading));+ self.pos = (end.x, end.y);+ self.heading = end_heading;+ self.path = None;+ self.dist = 0.0;+ } else {+ let s: PathState = p.sample(self.dist);+ self.pos = (s.x, s.y);+ let desired = s.heading + std::f64::consts::PI;+ let max_step = 14.0 * dt;+ self.heading = rotate_toward(self.heading, desired, max_step);+ }+ } else if let Some(mut s) = self.spring {+ if let Some((tx, ty, th)) = self.spring_tgt {+ let sdt = dt / 4.0;+ for _ in 0..4 {+ s.vx += (-spring_k * s.ox - spring_c * s.vx) * sdt;+ s.vy += (-spring_k * s.oy - spring_c * s.vy) * sdt;+ s.ox += s.vx * sdt;+ s.oy += s.vy * sdt;+ }+ self.pos = (tx + s.ox, ty + s.oy);+ self.heading = th;+ if s.ox.hypot(s.oy) < 0.3 && s.vx.hypot(s.vy) < 2.0 {+ self.pos = (tx, ty);+ self.spring = None;+ } else {+ self.spring = Some(s);+ }+ }+ }++ if let Some(t) = self.click_t {+ let next = t + dt * 4.0;+ self.click_t = if next >= 1.0 { None } else { Some(next) };+ }++ let idle_hide_ms = self.motion.idle_hide_ms;+ if idle_hide_ms > 0.0 {+ let moving = self.path.is_some() || self.spring.is_some() || self.click_t.is_some();+ if moving {+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ } else {+ self.idle_secs += dt;+ let fade_start = idle_hide_ms / 1000.0;+ let fade_end = fade_start + 0.18;+ if self.idle_secs > fade_end {+ self.idle_alpha = 0.0;+ } else if self.idle_secs > fade_start {+ let t = (self.idle_secs - fade_start) / 0.18;+ self.idle_alpha = 1.0 - t.clamp(0.0, 1.0);+ }+ }+ } else {+ self.idle_alpha = 1.0;+ }+ }++ fn apply_command(&mut self, cmd: OverlayCommand) {+ match cmd {+ OverlayCommand::MoveTo { x, y, end_heading_radians } => {+ let (x0, y0) = self.pos;+ let th0 = self.heading + std::f64::consts::PI;+ let th1 = end_heading_radians + std::f64::consts::PI;+ const CLICK_OFFSET: f64 = 16.0;+ const TURN_RADIUS: f64 = 80.0;+ let tx = x + end_heading_radians.cos() * CLICK_OFFSET;+ let ty = y + end_heading_radians.sin() * CLICK_OFFSET;+ let plan = PathPlanner::plan(+ x0, y0, th0,+ tx, ty, th1,+ end_heading_radians,+ TURN_RADIUS,+ );+ self.path = Some(plan);+ self.dist = 0.0;+ self.start_t = Instant::now();+ self.spring = None;+ self.spring_tgt = None;+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ }+ OverlayCommand::ClickPulse { x, y } => {+ self.pos = (x, y);+ self.click_t = Some(0.0);+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ }+ OverlayCommand::SetEnabled(v) => { self.visible = v; }+ OverlayCommand::SetMotion(m) => { self.motion = m; }+ OverlayCommand::SetPalette(p) => { self.palette = p; }+ OverlayCommand::PinAbove(wid) => { self.pinned_wid = Some(wid); }+ // Custom cursor shape/gradient/focus-rect — not yet rendered on Linux;+ // accepted silently so the tool doesn't return an error.+ OverlayCommand::SetShape(_) | OverlayCommand::SetGradient { .. }+ | OverlayCommand::ShowFocusRect(_) => {}+ }+ }+}++fn rotate_toward(current: f64, desired: f64, max_step: f64) -> f64 {+ let mut diff = desired - current;+ while diff > std::f64::consts::PI { diff -= 2.0 * std::f64::consts::PI; }+ while diff < -std::f64::consts::PI { diff += 2.0 * std::f64::consts::PI; }+ current + diff.clamp(-max_step, max_step)+}++// ── Renderer (shared tiny-skia logic) ─────────────────────────────────────++fn render_frame(rs: &RenderState) -> tiny_skia::Pixmap {+ let w = rs.scr_w.max(1);+ let h = rs.scr_h.max(1);+ let mut pm = tiny_skia::Pixmap::new(w, h)+ .unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap());++ if !rs.visible || rs.pos.0 < -100.0 || rs.idle_alpha < 0.004 {+ return pm;+ }++ let (px, py) = rs.pos;+ let heading = rs.heading;+ let alpha_scale = rs.idle_alpha as f32;++ let bloom_r: f32 = 22.0;+ let [br, bg, bb, _] = rs.palette.bloom_inner;+ let bloom_inner = tiny_skia::Color::from_rgba8(br, bg, bb, (115.0 * alpha_scale) as u8);+ let [or_, og, ob, _] = rs.palette.bloom_outer;+ let bloom_outer = tiny_skia::Color::from_rgba8(or_, og, ob, (26.0 * alpha_scale) as u8);+ let bloom_zero = tiny_skia::Color::from_rgba8(or_, og, ob, 0);++ let bloom_paint = {+ let mut p = tiny_skia::Paint::default();+ p.shader = tiny_skia::RadialGradient::new(+ tiny_skia::Point::from_xy(px as f32, py as f32),+ tiny_skia::Point::from_xy(px as f32, py as f32),+ bloom_r,+ vec![+ tiny_skia::GradientStop::new(0.0, bloom_inner),+ tiny_skia::GradientStop::new(0.5, bloom_outer),+ tiny_skia::GradientStop::new(1.0, bloom_zero),+ ],+ tiny_skia::SpreadMode::Pad,+ tiny_skia::Transform::identity(),+ ).unwrap_or(tiny_skia::Shader::SolidColor(bloom_inner));+ p.anti_alias = true;+ p+ };+ if let Some(r) = tiny_skia::Rect::from_xywh(+ (px - bloom_r as f64) as f32, (py - bloom_r as f64) as f32,+ bloom_r * 2.0, bloom_r * 2.0,+ ) {+ pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None);+ }++ if let Some(t) = rs.click_t {+ let ring_r = (bloom_r + 20.0 * t as f32) * (1.0 - t as f32 * 0.5);+ let alpha = ((1.0 - t) * 180.0 * alpha_scale as f64) as u8;+ let [cr, cg, cb, _] = rs.palette.cursor_mid;+ let ring_color = tiny_skia::Color::from_rgba8(cr, cg, cb, alpha);+ let mut ring_paint = tiny_skia::Paint::default();+ ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color);+ ring_paint.anti_alias = true;+ let stroke = tiny_skia::Stroke { width: 2.0, ..Default::default() };+ let mut pb = tiny_skia::PathBuilder::new();+ pb.push_circle(px as f32, py as f32, ring_r);+ if let Some(path) = pb.finish() {+ pm.stroke_path(&path, &ring_paint, &stroke, tiny_skia::Transform::identity(), None);+ }+ }++ if let Some(ref shape) = rs.shape {+ let sz = 32.0_f32;+ if let Some(pix) = tiny_skia::PixmapRef::from_bytes(&shape.pixels, shape.width, shape.height) {+ let transform = tiny_skia::Transform::from_rotate_at(+ heading.to_degrees() as f32 + 180.0, px as f32, py as f32,+ ).pre_translate(px as f32 - sz / 2.0, py as f32 - sz / 2.0);+ let mut paint = tiny_skia::PixmapPaint::default();+ paint.opacity = alpha_scale;+ pm.draw_pixmap(0, 0, pix, &paint, transform, None);+ }+ } else {+ draw_default_arrow(&mut pm, &rs.palette, px as f32, py as f32, heading as f32, alpha_scale);+ }++ pm+}++fn draw_default_arrow(+ pm: &mut tiny_skia::Pixmap,+ palette: &Palette,+ px: f32, py: f32,+ heading: f32,+ alpha_scale: f32,+) {+ let verts: [(f32, f32); 4] = [(14.0, 0.0), (-8.0, -9.0), (-3.0, 0.0), (-8.0, 9.0)];+ let angle = heading + std::f64::consts::PI as f32;+ let (sa, ca) = (angle.sin(), angle.cos());+ let xform = |(vx, vy): (f32, f32)| (px + ca * vx - sa * vy, py + sa * vx + ca * vy);+ let pts: Vec<(f32, f32)> = verts.iter().map(|&v| xform(v)).collect();+ let mut pb = tiny_skia::PathBuilder::new();+ pb.move_to(pts[0].0, pts[0].1);+ for p in &pts[1..] { pb.line_to(p.0, p.1); }+ pb.close();+ let arrow_path = match pb.finish() { Some(p) => p, None => return };+ let tip = pts[0];+ let tail = ((pts[1].0 + pts[3].0) / 2.0, (pts[1].1 + pts[3].1) / 2.0);+ let [r0, g0, b0, _] = palette.cursor_start;+ let [r1, g1, b1, _] = palette.cursor_mid;+ let [r2, g2, b2, _] = palette.cursor_end;+ let a = (255.0 * alpha_scale) as u8;+ let fill_paint = {+ let mut p = tiny_skia::Paint::default();+ p.shader = tiny_skia::LinearGradient::new(+ tiny_skia::Point::from_xy(tip.0, tip.1),+ tiny_skia::Point::from_xy(tail.0, tail.1),+ vec![+ tiny_skia::GradientStop::new(0.00, tiny_skia::Color::from_rgba8(r0, g0, b0, a)),+ tiny_skia::GradientStop::new(0.53, tiny_skia::Color::from_rgba8(r1, g1, b1, a)),+ tiny_skia::GradientStop::new(1.00, tiny_skia::Color::from_rgba8(r2, g2, b2, a)),+ ],+ tiny_skia::SpreadMode::Pad,+ tiny_skia::Transform::identity(),+ ).unwrap_or(tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(r1, g1, b1, a)));+ p.anti_alias = true;+ p+ };+ pm.fill_path(&arrow_path, &fill_paint, tiny_skia::FillRule::Winding,+ tiny_skia::Transform::identity(), None);+ let mut sp = tiny_skia::Paint::default();+ sp.shader = tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(255, 255, 255, a));+ sp.anti_alias = true;+ let stroke = tiny_skia::Stroke { width: 1.5, ..Default::default() };+ pm.stroke_path(&arrow_path, &sp, &stroke, tiny_skia::Transform::identity(), None);+}++// ── X11 thread ────────────────────────────────────────────────────────────++#[cfg(target_os = "linux")]+fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayCommand>) {+ use x11rb::connection::Connection;+ use x11rb::protocol::xproto::*;+ use x11rb::protocol::xproto::ConnectionExt as _;+ use x11rb::protocol::shape::*;+ use x11rb::protocol::shape::ConnectionExt as _;+ use x11rb::wrapper::ConnectionExt as _;+ use x11rb::COPY_FROM_PARENT;++ // Connect to X11.+ let (conn, screen_num) = match x11rb::connect(None) {+ Ok(c) => c,+ Err(e) => {+ tracing::warn!("X11 overlay: cannot connect to display: {e}");+ return;+ }+ };++ let screen = &conn.setup().roots[screen_num];+ let root = screen.root;+ let scr_w = screen.width_in_pixels as u32;+ let scr_h = screen.height_in_pixels as u32;++ // Update render state with screen size.+ {+ let mut guard = RENDER.lock().unwrap();+ if let Some(rs) = guard.as_mut() {+ rs.scr_w = scr_w;+ rs.scr_h = scr_h;+ }+ }++ // Find 32-bit ARGB visual for compositing.+ // Falls back to the default visual if XComposite 32-bit isn't available.+ let (visual_id, depth, colormap) = find_argb_visual(&conn, screen)+ .unwrap_or((screen.root_visual, screen.root_depth, screen.default_colormap));++ // Create a matching colormap if we got a non-default visual.+ let colormap = if visual_id != screen.root_visual {+ let cm = conn.generate_id().unwrap();+ conn.create_colormap(ColormapAlloc::NONE, cm, root, visual_id).ok();+ cm+ } else {+ colormap+ };++ // Create the overlay window.+ let win = conn.generate_id().unwrap();+ let win_aux = CreateWindowAux::new()+ .background_pixel(0)+ .border_pixel(0)+ .colormap(colormap)+ // Override-redirect = no window manager decoration, no focus.+ .override_redirect(1u32)+ // Input passthrough: do not receive button/key events.+ .event_mask(EventMask::NO_EVENT);++ conn.create_window(+ depth, win, root,+ 0, 0, scr_w as u16, scr_h as u16,+ 0,+ WindowClass::INPUT_OUTPUT,+ visual_id,+ &win_aux,+ ).ok();++ // Set window title (identifies our overlay, matches Windows convention).+ let title = format!("TropeCUA.AgentCursorOverlay.{}", cfg.cursor_id);+ conn.change_property8(+ PropMode::REPLACE, win,+ AtomEnum::WM_NAME,+ AtomEnum::STRING,+ title.as_bytes(),+ ).ok();++ // Make the window fully click-through using the Shape extension (empty input region).+ // This is the X11 equivalent of WS_EX_TRANSPARENT on Windows.+ let empty_region_pixmap = conn.generate_id().unwrap();+ conn.create_pixmap(1, empty_region_pixmap, root, 1, 1).ok();+ conn.shape_mask(+ x11rb::protocol::shape::SO::SET,+ x11rb::protocol::shape::SK::INPUT,+ win,+ 0, 0,+ x11rb::NONE,+ ).ok();+ conn.free_pixmap(empty_region_pixmap).ok();++ conn.map_window(win).ok();+ conn.flush().ok();++ // Main render loop at ~60Hz.+ let frame_dur = Duration::from_millis(16);+ let mut last_tick = Instant::now();+ let mut last_ztick = Instant::now();++ loop {+ let now = Instant::now();+ let dt = now.duration_since(last_tick).as_secs_f64().min(0.05);+ last_tick = now;++ // Drain commands and tick.+ {+ let mut guard = RENDER.lock().unwrap();+ if let Some(rs) = guard.as_mut() {+ while let Ok(cmd) = rx.try_recv() {+ rs.apply_command(cmd);+ }+ rs.tick(dt);+ }+ }++ // Render and paint.+ let pixmap = {+ let guard = RENDER.lock().unwrap();+ guard.as_ref().map(|rs| render_frame(rs))+ };++ if let Some(pm) = pixmap {+ paint_x11(&conn, win, scr_w, scr_h, depth, visual_id, &pm);+ }++ // Z-order maintenance every 80ms.+ if last_ztick.elapsed() >= Duration::from_millis(80) {+ last_ztick = Instant::now();+ let pinned_wid = {+ let guard = RENDER.lock().unwrap();+ guard.as_ref().and_then(|rs| rs.pinned_wid)+ };+ let aux = if let Some(target_xid) = pinned_wid {+ // Place overlay just above the pinned X11 window.+ ConfigureWindowAux::new()+ .sibling(target_xid as u32)+ .stack_mode(StackMode::ABOVE)+ } else {+ ConfigureWindowAux::new().stack_mode(StackMode::ABOVE)+ };+ conn.configure_window(win, &aux).ok();+ conn.flush().ok();+ }++ // Drain any X events (needed to avoid blocking).+ while let Ok(Some(_)) = conn.poll_for_event() {}++ let elapsed = Instant::now().duration_since(last_tick);+ if let Some(remaining) = frame_dur.checked_sub(elapsed) {+ std::thread::sleep(remaining);+ }+ }+}++#[cfg(target_os = "linux")]+fn find_argb_visual(+ conn: &impl x11rb::connection::Connection,+ screen: &x11rb::protocol::xproto::Screen,+) -> Option<(u32, u8, u32)> {+ use x11rb::protocol::xproto::VisualClass;+ // Walk all depth entries looking for a 32-bit ARGB visual.+ for depth_entry in &screen.allowed_depths {+ if depth_entry.depth != 32 { continue; }+ for visual in &depth_entry.visuals {+ if visual.class == VisualClass::TRUE_COLOR {+ return Some((visual.visual_id, 32, screen.default_colormap));+ }+ }+ }+ None+}++/// Blit a tiny-skia pixmap to the X11 window using XPutImage (ZPixmap).+/// The pixmap is premultiplied RGBA; X11 ARGB is premultiplied BGRA.+#[cfg(target_os = "linux")]+fn paint_x11(+ conn: &impl x11rb::connection::Connection,+ win: u32,+ w: u32, h: u32,+ depth: u8,+ _visual_id: u32,+ pm: &tiny_skia::Pixmap,+) {+ use x11rb::protocol::xproto::*;+ use x11rb::protocol::xproto::ConnectionExt as _;+ if pm.width() == 0 || pm.height() == 0 { return; }++ // Create a GC for the window if we don't have one.+ // (Simplified: we recreate it every frame which is safe but not optimal.)+ let gc_id = match conn.generate_id() {+ Ok(id) => id,+ Err(_) => return,+ };+ let gc_aux = CreateGCAux::new();+ if conn.create_gc(gc_id, win, &gc_aux).is_err() { return; }++ // Convert RGBA premult → BGRA premult for X11.+ let src = pm.data();+ let mut bgra: Vec<u8> = Vec::with_capacity(src.len());+ for chunk in src.chunks_exact(4) {+ bgra.push(chunk[2]); // B+ bgra.push(chunk[1]); // G+ bgra.push(chunk[0]); // R+ bgra.push(chunk[3]); // A+ }++ // XPutImage (ZPixmap).+ let _ = conn.put_image(+ ImageFormat::Z_PIXMAP,+ win,+ gc_id,+ w as u16, h as u16,+ 0, 0,+ 0,+ depth,+ &bgra,+ );++ conn.free_gc(gc_id).ok();+ conn.flush().ok();+}++#[cfg(not(target_os = "linux"))]+fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver<OverlayCommand>) {}diff --git a/libs/cua-driver-rs/crates/platform-linux/src/proc_fs.rs b/libs/cua-driver-rs/crates/platform-linux/src/proc_fs.rsnew file mode 100644index 0000000000..16b758855f--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/proc_fs.rs@@ -0,0 +1,58 @@+//! Linux process enumeration via the /proc filesystem.+//!+//! Each /proc/<pid>/status file contains Name:, Pid:, PPid: etc.+//! /proc/<pid>/cmdline is the full command line (NUL-separated).++use std::fs;+use std::path::Path;++#[derive(Debug, Clone)]+pub struct ProcessInfo {+ pub pid: u32,+ pub name: String,+ pub cmdline: String,+}++/// Return all running processes by reading /proc/<pid>/status.+pub fn list_processes() -> Vec<ProcessInfo> {+ let mut result = Vec::new();+ let proc_dir = Path::new("/proc");+ let entries = match fs::read_dir(proc_dir) {+ Ok(e) => e,+ Err(_) => return result,+ };++ for entry in entries.flatten() {+ let name = entry.file_name();+ let pid_str = name.to_string_lossy();+ let pid: u32 = match pid_str.parse() {+ Ok(p) => p,+ Err(_) => continue, // Skip non-numeric entries.+ };++ let status_path = proc_dir.join(&*pid_str).join("status");+ let status = match fs::read_to_string(&status_path) {+ Ok(s) => s,+ Err(_) => continue,+ };++ let proc_name = status.lines()+ .find(|l| l.starts_with("Name:"))+ .map(|l| l[5..].trim().to_owned())+ .unwrap_or_default();++ let cmdline_path = proc_dir.join(&*pid_str).join("cmdline");+ let cmdline = fs::read(cmdline_path).ok()+ .map(|b| {+ // cmdline is NUL-separated; first entry is argv[0].+ let s = String::from_utf8_lossy(&b);+ s.split('\0').next().unwrap_or("").trim().to_owned()+ })+ .unwrap_or_default();++ result.push(ProcessInfo { pid, name: proc_name, cmdline });+ }++ result.sort_by_key(|p| p.pid);+ result+}diff --git a/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rsnew file mode 100644index 0000000000..77daee7832--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs@@ -0,0 +1,1797 @@+//! Real Linux tool implementations (compiled only on Linux).++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};+use serde_json::{json, Value};+use std::sync::{Arc, RwLock};++use crate::atspi::ElementCache;+use cursor_overlay::CursorRegistry;++// ── DriverConfig + ResizeRegistry + ZoomRegistry ─────────────────────────────++#[derive(Clone)]+pub struct DriverConfig {+ pub capture_mode: String,+ pub max_image_dimension: u32,+}++impl Default for DriverConfig {+ fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } }+}++pub struct ResizeRegistry {+ ratios: std::sync::Mutex<std::collections::HashMap<u32, f64>>,+}++impl ResizeRegistry {+ pub fn new() -> Self { Self { ratios: std::sync::Mutex::new(Default::default()) } }+ pub fn set_ratio(&self, pid: u32, ratio: f64) { self.ratios.lock().unwrap().insert(pid, ratio); }+ pub fn clear_ratio(&self, pid: u32) { self.ratios.lock().unwrap().remove(&pid); }+ pub fn ratio(&self, pid: u32) -> Option<f64> { self.ratios.lock().unwrap().get(&pid).copied() }+}++/// Per-process zoom context — stores padded crop origin and resize scale from+/// the most recent `zoom` call so `click(from_zoom=true)` can translate+/// zoom-image pixel coordinates back to full-window coordinates.+#[derive(Clone, Copy, Debug)]+pub struct ZoomContext {+ pub origin_x: f64,+ pub origin_y: f64,+ /// Inverse resize scale: `cw / out_w` (1.0 = no downscale).+ pub scale_inv: f64,+}++impl ZoomContext {+ pub fn zoom_to_window(&self, px: f64, py: f64) -> (f64, f64) {+ (self.origin_x + px * self.scale_inv, self.origin_y + py * self.scale_inv)+ }+}++pub struct ZoomRegistry {+ inner: std::sync::Mutex<std::collections::HashMap<u32, ZoomContext>>,+}++impl ZoomRegistry {+ pub fn new() -> Self { Self { inner: std::sync::Mutex::new(Default::default()) } }+ pub fn set(&self, pid: u32, ctx: ZoomContext) { self.inner.lock().unwrap().insert(pid, ctx); }+ pub fn get(&self, pid: u32) -> Option<ZoomContext> { self.inner.lock().unwrap().get(&pid).copied() }+}++pub struct ToolState {+ pub element_cache: Arc<ElementCache>,+ pub cursor_registry: Arc<CursorRegistry>,+ pub resize_registry: Arc<ResizeRegistry>,+ pub zoom_registry: Arc<ZoomRegistry>,+ pub config: Arc<RwLock<DriverConfig>>,+}++impl ToolState {+ pub fn new() -> Arc<Self> {+ Arc::new(Self {+ element_cache: Arc::new(ElementCache::new()),+ cursor_registry: Arc::new(CursorRegistry::new()),+ resize_registry: Arc::new(ResizeRegistry::new()),+ zoom_registry: Arc::new(ZoomRegistry::new()),+ config: Arc::new(RwLock::new(DriverConfig::default())),+ })+ }+}++// ── list_apps ────────────────────────────────────────────────────────────────++pub struct ListAppsTool;+static LIST_APPS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ListAppsTool {+ fn def(&self) -> &ToolDef {+ LIST_APPS_DEF.get_or_init(|| ToolDef {+ name: "list_apps".into(),+ description: "List running processes via /proc filesystem.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, _args: Value) -> ToolResult {+ let procs = tokio::task::spawn_blocking(|| crate::proc_fs::list_processes()).await.unwrap_or_default();+ let mut lines = vec![format!("Found {} processes:", procs.len())];+ for p in &procs {+ let cmd = if p.cmdline.is_empty() { p.name.clone() } else { p.cmdline.clone() };+ lines.push(format!(" {} (pid {})", cmd, p.pid));+ }+ let structured = json!({ "processes": procs.iter().map(|p| json!({+ "pid": p.pid, "name": p.name, "cmdline": p.cmdline+ })).collect::<Vec<_>>() });+ ToolResult::text(lines.join("\n")).with_structured(structured)+ }+}++// ── list_windows ─────────────────────────────────────────────────────────────++pub struct ListWindowsTool;+static LIST_WINDOWS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ListWindowsTool {+ fn def(&self) -> &ToolDef {+ LIST_WINDOWS_DEF.get_or_init(|| ToolDef {+ name: "list_windows".into(),+ description: "List top-level X11 windows via _NET_CLIENT_LIST.".into(),+ input_schema: json!({"type":"object","properties":{+ "pid":{"type":"integer"},+ "on_screen_only":{"type":"boolean","description":"When true, filter to visible windows only. Default false."}+ },"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let filter_pid = args.get("pid").and_then(|v| v.as_u64()).map(|v| v as u32);+ let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(filter_pid)).await.unwrap_or_default();+ let mut lines = vec![format!("Found {} windows:", windows.len())];+ for w in &windows {+ lines.push(format!(" [xid={}] pid={:?} \"{}\" {}x{}+{}+{}",+ w.xid, w.pid, w.title, w.width, w.height, w.x, w.y));+ }+ let structured = json!({ "windows": windows.iter().map(|w| json!({+ "window_id": w.xid,+ "pid": w.pid,+ "title": w.title,+ "x": w.x, "y": w.y,+ "width": w.width, "height": w.height,+ })).collect::<Vec<_>>() });+ ToolResult::text(lines.join("\n")).with_structured(structured)+ }+}++// ── get_window_state ─────────────────────────────────────────────────────────++pub struct GetWindowStateTool {+ state: Arc<ToolState>,+}++static GWS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetWindowStateTool {+ fn def(&self) -> &ToolDef {+ GWS_DEF.get_or_init(|| ToolDef {+ name: "get_window_state".into(),+ description: "Walk a running app's AT-SPI tree and return a Markdown rendering of its UI. Also captures a screenshot.".into(),+ input_schema: json!({"type":"object","required":["pid","window_id"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer","description":"X11 XID from list_windows."},+ "capture_mode":{"type":"string","enum":["som","vision","ax"],+ "description":"som=tree+screenshot (default), vision=screenshot only, ax=tree only."},+ "query":{"type":"string"}+ },"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let xid = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: window_id"),+ };+ let (default_mode, max_dim) = {+ let cfg = self.state.config.read().unwrap();+ (cfg.capture_mode.clone(), cfg.max_image_dimension)+ };+ let capture_mode = args.get("capture_mode").and_then(|v| v.as_str())+ .unwrap_or(&default_mode).to_owned();+ let query = args.get("query").and_then(|v| v.as_str()).map(str::to_owned);++ // "ax" = tree only; "vision" = screenshot only; "som" (default) = both.+ let do_tree = capture_mode != "vision";+ let do_shot = capture_mode != "ax";+ let state = self.state.clone();++ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {+ let tree_result = if do_tree {+ Some(crate::atspi::walk_tree(pid, xid, query.as_deref()))+ } else {+ None+ };+ let screenshot = if do_shot {+ match crate::capture::screenshot_window_bytes(xid) {+ Ok(raw) => {+ let orig_w = crate::capture::png_dimensions_pub(&raw).map(|(w, _)| w).unwrap_or(0);+ let png = crate::capture::resize_png_if_needed(&raw, max_dim)?;+ let (w, h) = crate::capture::png_dimensions_pub(&png)?;+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};+ let original_w = if w < orig_w { Some(orig_w) } else { None };+ Some((B64.encode(&png), w, h, original_w))+ }+ Err(_) => None,+ }+ } else {+ None+ };+ Ok((tree_result, screenshot))+ }).await;++ match result {+ Ok(Ok((tree_opt, shot_opt))) => {+ let mut content = Vec::new();+ let mut structured = json!({ "window_id": xid, "pid": pid });++ if let Some(tr) = tree_opt {+ let count = tr.nodes.iter().filter(|n| n.element_index.is_some()).count();+ let header = format!("window_id={xid} pid={pid} elements={count}\n\n");+ content.push(mcp_server::protocol::Content::text(header + &tr.tree_markdown));+ state.element_cache.update(pid, xid, &tr.nodes);+ structured["element_count"] = json!(count);+ structured["tree_markdown"] = json!(tr.tree_markdown);+ }++ if let Some((b64, w, h, orig_w)) = shot_opt {+ if let Some(ow) = orig_w {+ if w > 0 { state.resize_registry.set_ratio(pid, ow as f64 / w as f64); }+ } else {+ state.resize_registry.clear_ratio(pid);+ }+ content.push(mcp_server::protocol::Content::image_png(b64));+ structured["screenshot_width"] = json!(w);+ structured["screenshot_height"] = json!(h);+ }++ ToolResult { content, is_error: None, structured_content: Some(structured) }+ }+ Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── launch_app ───────────────────────────────────────────────────────────────++pub struct LaunchAppTool;+static LAUNCH_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for LaunchAppTool {+ fn def(&self) -> &ToolDef {+ LAUNCH_DEF.get_or_init(|| ToolDef {+ name: "launch_app".into(),+ description: "Launch a Linux app in the background. Provide name (app name, launched via \+ xdg-open or direct exec), bundle_id (ignored on Linux), or urls (list of URLs to open).".into(),+ input_schema: json!({"type":"object","properties":{+ "name":{"type":"string","description":"App name or command to launch."},+ "bundle_id":{"type":"string","description":"Ignored on Linux (macOS/Windows concept)."},+ "urls":{"type":"array","items":{"type":"string"},"description":"URLs to open via xdg-open."}+ },"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let name_opt = args.get("name").and_then(|v| v.as_str()).map(str::to_owned);+ let urls: Vec<String> = args.get("urls").and_then(|v| v.as_array())+ .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();++ if name_opt.is_none() && urls.is_empty() {+ return ToolResult::error("Provide at least one of: name or urls.");+ }++ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {+ // Open URLs via xdg-open.+ if !urls.is_empty() {+ for url in &urls {+ std::process::Command::new("xdg-open").arg(url).spawn()?;+ }+ return Ok(format!("Opened {} URL(s) via xdg-open.", urls.len()));+ }+ // Launch by name — try direct exec, then xdg-open.+ if let Some(name) = name_opt {+ let mut parts = name.split_whitespace();+ let prog = parts.next().unwrap_or(&name);+ let rest: Vec<&str> = parts.collect();+ match std::process::Command::new(prog).args(&rest).spawn() {+ Ok(child) => return Ok(format!("Launched '{}' with pid {}.", name, child.id())),+ Err(_) => {+ // Fall back to xdg-open for .desktop app names.+ std::process::Command::new("xdg-open").arg(&name).spawn()?;+ return Ok(format!("Opened '{}' via xdg-open.", name));+ }+ }+ }+ unreachable!()+ }).await;++ match result {+ Ok(Ok(msg)) => ToolResult::text(msg),+ Ok(Err(e)) => ToolResult::error(format!("Failed to launch: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── shared helpers ────────────────────────────────────────────────────────────++/// Resolve an AT-SPI element's center in window-local X11 coordinates.+///+/// Returns `(xid, window_local_x, window_local_y)`.+/// Looks up element bounds via pyatspi subprocess, finds the owning window+/// (via `xid_hint` or the first window for `pid`), then converts screen-absolute+/// → window-local coords via X11 translate_coordinates.+fn resolve_element_local_coords(pid: u32, idx: usize, xid_hint: Option<u64>)+ -> anyhow::Result<(u64, f64, f64)>+{+ let (bx, by, bw, bh) = crate::atspi::get_element_bounds(pid, idx)?;+ let screen_cx = bx as f64 + bw as f64 / 2.0;+ let screen_cy = by as f64 + bh as f64 / 2.0;++ let xid = if let Some(x) = xid_hint {+ x+ } else {+ crate::x11::list_windows(Some(pid))+ .into_iter().next().map(|w| w.xid)+ .ok_or_else(|| anyhow::anyhow!("No windows for pid {pid}"))?+ };++ use x11rb::connection::Connection;+ use x11rb::protocol::xproto::ConnectionExt as _;+ use x11rb::rust_connection::RustConnection;+ let (conn, screen_num) = RustConnection::connect(None)?;+ let root = conn.setup().roots[screen_num].root;+ let reply = conn.translate_coordinates(xid as u32, root, 0, 0)?.reply()?;+ let local_x = screen_cx - reply.dst_x as f64;+ let local_y = screen_cy - reply.dst_y as f64;+ Ok((xid, local_x, local_y))+}++// ── click ─────────────────────────────────────────────────────────────────────++pub struct ClickTool {+ state: Arc<ToolState>,+}+static CLICK_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ClickTool {+ fn def(&self) -> &ToolDef {+ CLICK_DEF.get_or_init(|| ToolDef {+ name: "click".into(),+ description: "Click at (x,y) coordinates or an element_index (AT-SPI) in a window via \+ XSendEvent. No focus steal. Provide either (window_id + x/y) or (pid + element_index). \+ After a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to \+ full-window space.".into(),+ input_schema: json!({+ "type":"object","required":["pid"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "x":{"type":"number"},+ "y":{"type":"number"},+ "element_index":{"type":"integer","description":"AT-SPI element index from get_window_state."},+ "button":{"type":"string","enum":["left","right","middle"]},+ "count":{"type":"integer"},+ "from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(1) as usize;+ let button: u8 = match args.get("button").and_then(|v| v.as_str()).unwrap_or("left") {+ "right" => 3,+ "middle" => 2,+ _ => 1,+ };++ if let Some(idx) = args.get("element_index").and_then(|v| v.as_u64()) {+ let idx = idx as usize;+ let xid_hint = args.get("window_id").and_then(|v| v.as_u64());+ // For element_index: try AT-SPI perform_action first (background-safe).+ // Always get bounds to send the overlay ClickPulse at the element center.+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(f64, f64)> {+ // Get element screen-absolute center for the overlay pulse.+ let screen_cx;+ let screen_cy;+ if let Ok((bx, by, bw, bh)) = crate::atspi::get_element_bounds(pid, idx) {+ screen_cx = bx as f64 + bw as f64 / 2.0;+ screen_cy = by as f64 + bh as f64 / 2.0;+ } else {+ screen_cx = 0.0;+ screen_cy = 0.0;+ }++ // Primary: AT-SPI doAction(0) — typically "click", no focus steal.+ if crate::atspi::perform_action(pid, idx).is_ok() {+ return Ok((screen_cx, screen_cy));+ }++ // Fallback: XSendEvent at window-local coords.+ let (xid, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?;+ crate::input::send_click(xid, lx as i32, ly as i32, count, button)?;+ Ok((screen_cx, screen_cy))+ }).await;+ return match result {+ Ok(Ok((x, y))) => {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y });+ ToolResult::text(format!("Clicked element [{idx}] (pid {pid})."))+ }+ Ok(Err(e)) => ToolResult::error(format!("AT-SPI element click failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }++ // Coordinate-based path.+ let xid = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v,+ None => return ToolResult::error("Provide either element_index or window_id + x/y."),+ };+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);+ let mut x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let mut y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);+ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; }+ None => return ToolResult::error(+ format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")+ ),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ x *= ratio;+ y *= ratio;+ }++ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y });+ // Pin overlay just above the target window for z-order sandwich.+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));++ let (xi, yi) = (x as i32, y as i32);+ let result = tokio::task::spawn_blocking(move || {+ crate::input::send_click(xid, xi, yi, count, button)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("✅ Clicked at ({x:.1}, {y:.1}) × {count}.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── type_text ─────────────────────────────────────────────────────────────────++pub struct TypeTextTool;+static TYPE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for TypeTextTool {+ fn def(&self) -> &ToolDef {+ TYPE_DEF.get_or_init(|| ToolDef {+ name: "type_text".into(),+ description: "Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal.".into(),+ input_schema: json!({+ "type":"object","required":["pid","text"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "text":{"type":"string"},+ "element_index":{"type":"integer","description":"Element index from get_window_state (accepted for cross-platform parity)."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32;+ let text = match args.get("text").and_then(|v| v.as_str()) {+ Some(t) => t.to_owned(),+ None => return ToolResult::error("Missing required parameter: text"),+ };+ let xid_opt = args.get("window_id").and_then(|v| v.as_u64());++ // Resolve XID: use window_id if given, else first window for pid.+ let xid = match xid_opt {+ Some(x) => x,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.xid,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };+ let text_len = text.chars().count();+ let result = tokio::task::spawn_blocking(move || {+ crate::input::send_type_text(xid, &text)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s).")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── press_key ─────────────────────────────────────────────────────────────────++pub struct PressKeyTool;+static PRESS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for PressKeyTool {+ fn def(&self) -> &ToolDef {+ PRESS_DEF.get_or_init(|| ToolDef {+ name: "press_key".into(),+ description: "Press a key via XSendEvent to a window. No focus steal.".into(),+ input_schema: json!({+ "type":"object","required":["pid","key"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "key":{"type":"string"},+ "modifiers":{"type":"array","items":{"type":"string"}}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32;+ let key = match args.get("key").and_then(|v| v.as_str()) {+ Some(k) => k.to_owned(),+ None => return ToolResult::error("Missing required parameter: key"),+ };+ let mods: Vec<String> = args.get("modifiers")+ .and_then(|v| v.as_array())+ .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();+ let xid_opt = args.get("window_id").and_then(|v| v.as_u64());+ let xid = match xid_opt {+ Some(x) => x,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.xid,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };+ let key_for_task = key.clone();+ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = mods.iter().map(String::as_str).collect();+ crate::input::send_key(xid, &key_for_task, &m)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("Pressed key '{key}'.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── hotkey ────────────────────────────────────────────────────────────────────++fn is_modifier(k: &str) -> bool {+ matches!(k.to_lowercase().as_str(),+ "ctrl" | "control" | "shift" | "alt" | "super" | "meta" | "cmd" | "command" | "win" | "windows")+}++pub struct HotkeyTool;+static HOTKEY_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for HotkeyTool {+ fn def(&self) -> &ToolDef {+ HOTKEY_DEF.get_or_init(|| ToolDef {+ name: "hotkey".into(),+ description: "Press a combination of keys simultaneously, e.g. [\"ctrl\",\"c\"] for Copy. \+ Sent via XSendEvent directly to the target pid; target does NOT need to be frontmost.".into(),+ input_schema: json!({+ "type":"object","required":["pid","keys"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "keys":{"type":"array","items":{"type":"string"},"minItems":2,+ "description":"Modifier(s) + one non-modifier key, e.g. [\"ctrl\",\"c\"]."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32;+ let xid_opt = args.get("window_id").and_then(|v| v.as_u64());++ // Resolve XID: use window_id if given, else first window for pid.+ let xid = match xid_opt {+ Some(x) => x,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.xid,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };++ // Parse keys array (preferred) or fall back to legacy key+modifiers.+ let (key, mods) = if let Some(arr) = args.get("keys").and_then(|v| v.as_array()) {+ let keys: Vec<String> = arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect();+ let modifiers: Vec<String> = keys.iter().filter(|k| is_modifier(k)).cloned().collect();+ let non_mods: Vec<String> = keys.iter().filter(|k| !is_modifier(k)).cloned().collect();+ if non_mods.is_empty() {+ return ToolResult::error("keys must include at least one non-modifier key.");+ }+ (non_mods.last().unwrap().clone(), modifiers)+ } else if let Some(k) = args.get("key").and_then(|v| v.as_str()) {+ let mods: Vec<String> = args.get("modifiers").and_then(|v| v.as_array())+ .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();+ (k.to_owned(), mods)+ } else {+ return ToolResult::error("Provide 'keys' array (e.g. [\"ctrl\",\"c\"]) or 'key'+'modifiers' parameters.");+ };++ let key_display = format!("{}+{}", mods.join("+"), key);+ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = mods.iter().map(String::as_str).collect();+ crate::input::send_key(xid, &key, &m)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("Pressed {key_display} on pid {pid}.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── set_value ─────────────────────────────────────────────────────────────────++pub struct SetValueTool;+static SV_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetValueTool {+ fn def(&self) -> &ToolDef {+ SV_DEF.get_or_init(|| ToolDef {+ name: "set_value".into(),+ description: "Set value of an AT-SPI element via SetValue action.".into(),+ input_schema: json!({+ "type":"object","required":["pid","window_id","element_index","value"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "element_index":{"type":"integer"},+ "value":{"type":"string"}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let idx = match args.get("element_index").and_then(|v| v.as_u64()) {+ Some(v) => v as usize,+ None => return ToolResult::error("Missing required parameter: element_index"),+ };+ let value = match args.get("value").and_then(|v| v.as_str()) {+ Some(v) => v.to_owned(),+ None => return ToolResult::error("Missing required parameter: value"),+ };+ let value_for_task = value.clone();+ let result = tokio::task::spawn_blocking(move || {+ crate::atspi::set_value(pid, idx, &value_for_task)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("Set value of element [{idx}] to '{value}'.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── scroll ────────────────────────────────────────────────────────────────────++pub struct ScrollTool;+static SCROLL_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ScrollTool {+ fn def(&self) -> &ToolDef {+ SCROLL_DEF.get_or_init(|| ToolDef {+ name: "scroll".into(),+ description: "Scroll the target pid's focused region via XSendEvent Button4/5. \+ direction required; by defaults to line, amount defaults to 3.".into(),+ input_schema: json!({+ "type":"object","required":["pid","direction"],"properties":{+ "pid":{"type":"integer"},+ "direction":{"type":"string","enum":["up","down","left","right"]},+ "by":{"type":"string","enum":["line","page"]},+ "amount":{"type":"integer","minimum":1,"maximum":50},+ "window_id":{"type":"integer"},+ "element_index":{"type":"integer"}+ },"additionalProperties":false+ }),+ read_only: false, destructive: false, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let direction = match args.get("direction").and_then(|v| v.as_str()) {+ Some(d) => d.to_owned(),+ None => return ToolResult::error("Missing required parameter: direction"),+ };+ let amount = args.get("amount").and_then(|v| v.as_u64())+ .unwrap_or(3).clamp(1, 50) as usize;+ let xid_opt = args.get("window_id").and_then(|v| v.as_u64());++ // Resolve XID: use window_id if given, else first window for pid.+ let xid = match xid_opt {+ Some(x) => x,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.xid,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };++ // X11 scroll buttons: 4=up, 5=down, 6=left, 7=right+ // Note: "page" scroll is still per-click on X11; send more ticks for page.+ let button: u8 = match direction.as_str() {+ "up" => 4, "left" => 6, "right" => 7, _ => 5,+ };+ let result = tokio::task::spawn_blocking(move || {+ crate::input::send_click(xid, 0, 0, amount, button)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("Scrolled {direction} {amount} ticks.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── screenshot ────────────────────────────────────────────────────────────────++pub struct ScreenshotTool {+ state: Arc<ToolState>,+}+static SS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ScreenshotTool {+ fn def(&self) -> &ToolDef {+ SS_DEF.get_or_init(|| ToolDef {+ name: "screenshot".into(),+ description: "Capture a screenshot via XGetImage or `import` (ImageMagick). \+ Without window_id captures the full primary display. Supports png and jpeg formats.".into(),+ input_schema: json!({+ "type":"object","properties":{+ "window_id":{"type":"integer"},+ "format":{"type":"string","enum":["png","jpeg"]},+ "quality":{"type":"integer","minimum":1,"maximum":95}+ },"additionalProperties":false+ }),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let xid_opt = args.get("window_id").and_then(|v| v.as_u64());+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();+ let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;+ let is_jpeg = format == "jpeg";+ let max_dim = self.state.config.read().unwrap().max_image_dimension;++ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(String, u32, u32)> {+ let raw = match xid_opt {+ Some(xid) => crate::capture::screenshot_window_bytes(xid)?,+ None => crate::capture::screenshot_display_bytes()?,+ };+ let png_bytes = crate::capture::resize_png_if_needed(&raw, max_dim)?;+ if is_jpeg {+ let jpeg = crate::capture::png_bytes_to_jpeg(&png_bytes, quality)?;+ let (w, h) = crate::capture::png_dimensions_pub(&png_bytes)?;+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};+ Ok((B64.encode(&jpeg), w, h))+ } else {+ let (w, h) = crate::capture::png_dimensions_pub(&png_bytes)?;+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};+ Ok((B64.encode(&png_bytes), w, h))+ }+ }).await;++ match result {+ Ok(Ok((b64, w, h))) => {+ let label = if is_jpeg { "JPEG" } else { "PNG" };+ let scope = if xid_opt.is_some() { "window" } else { "display" };+ let mut tr = ToolResult::text(format!("Screenshot ({scope}): {w}×{h} {label}."));+ let img = if is_jpeg {+ mcp_server::protocol::Content::image_jpeg(b64)+ } else {+ mcp_server::protocol::Content::image_png(b64)+ };+ tr.content.push(img);+ tr.structured_content = Some(json!({ "width": w, "height": h, "format": label }));+ tr+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── double_click ──────────────────────────────────────────────────────────────++pub struct DoubleClickTool {+ state: Arc<ToolState>,+}+static DCLICK_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for DoubleClickTool {+ fn def(&self) -> &ToolDef {+ DCLICK_DEF.get_or_init(|| ToolDef {+ name: "double_click".into(),+ description: "Double-click at (x,y) or an element_index (AT-SPI bounds) via XSendEvent. \+ No focus steal. Provide either (window_id + x/y) or (pid + element_index). \+ After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.".into(),+ input_schema: json!({"type":"object","required":["pid"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "x":{"type":"number"},+ "y":{"type":"number"},+ "element_index":{"type":"integer","description":"AT-SPI element index from get_window_state."},+ "from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."}+ },"additionalProperties":false}),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ if let Some(idx) = args.get("element_index").and_then(|v| v.as_u64()) {+ let idx = idx as usize;+ let xid_hint = args.get("window_id").and_then(|v| v.as_u64());+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {+ resolve_element_local_coords(pid, idx, xid_hint)+ }).await;+ return match result {+ Ok(Ok((xid, lx, ly))) => {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: lx, y: ly });+ match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 2, 1)).await {+ Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked element [{idx}].")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+ Ok(Err(e)) => ToolResult::error(format!("AT-SPI bounds failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }+ let xid = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."),+ };+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);+ let mut x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let mut y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);+ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; }+ None => return ToolResult::error(+ format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")+ ),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ x *= ratio;+ y *= ratio;+ }+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y });+ let (xi, yi) = (x as i32, y as i32);+ let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 2, 1)).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked at ({x:.1}, {y:.1}).")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── right_click ───────────────────────────────────────────────────────────────++pub struct RightClickTool {+ state: Arc<ToolState>,+}+static RCLICK_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for RightClickTool {+ fn def(&self) -> &ToolDef {+ RCLICK_DEF.get_or_init(|| ToolDef {+ name: "right_click".into(),+ description: "Right-click at (x,y) or an element_index (AT-SPI bounds) via XSendEvent. \+ No focus steal. Provide either (window_id + x/y) or (pid + element_index). \+ After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.".into(),+ input_schema: json!({"type":"object","required":["pid"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "x":{"type":"number"},+ "y":{"type":"number"},+ "element_index":{"type":"integer","description":"AT-SPI element index from get_window_state."},+ "modifier":{"type":"array","items":{"type":"string"},"description":"Modifier keys to hold."},+ "from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."}+ },"additionalProperties":false}),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ if let Some(idx) = args.get("element_index").and_then(|v| v.as_u64()) {+ let idx = idx as usize;+ let xid_hint = args.get("window_id").and_then(|v| v.as_u64());+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {+ resolve_element_local_coords(pid, idx, xid_hint)+ }).await;+ return match result {+ Ok(Ok((xid, lx, ly))) => {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: lx, y: ly });+ match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 1, 3)).await {+ Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked element [{idx}].")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+ Ok(Err(e)) => ToolResult::error(format!("AT-SPI bounds failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }+ let xid = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."),+ };+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);+ let mut x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let mut y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);+ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; }+ None => return ToolResult::error(+ format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")+ ),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ x *= ratio;+ y *= ratio;+ }+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y });+ let (xi, yi) = (x as i32, y as i32);+ let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 1, 3)).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked at ({x:.1}, {y:.1}).")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── drag ─────────────────────────────────────────────────────────────────────++pub struct DragTool {+ state: Arc<ToolState>,+}+static DRAG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for DragTool {+ fn def(&self) -> &ToolDef {+ DRAG_DEF.get_or_init(|| ToolDef {+ name: "drag".into(),+ description: "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in \+ window-local screenshot pixels via XSendEvent (ButtonPress + MotionNotify × steps + ButtonRelease). \+ duration_ms (default 500), steps (default 20). No focus steal.".into(),+ input_schema: json!({"type":"object","required":["pid","from_x","from_y","to_x","to_y"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer","description":"Target window XID. Required."},+ "from_x":{"type":"number"},+ "from_y":{"type":"number"},+ "to_x":{"type":"number"},+ "to_y":{"type":"number"},+ "duration_ms":{"type":"integer","minimum":0,"maximum":10000,"description":"Total drag duration. Default: 500."},+ "steps":{"type":"integer","minimum":1,"maximum":200,"description":"Intermediate MotionNotify events. Default: 20."},+ "modifier":{"type":"array","items":{"type":"string"}},+ "button":{"type":"string","enum":["left","right","middle"],"description":"Mouse button. Default: left."},+ "from_zoom":{"type":"boolean"}+ },"additionalProperties":false}),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_u64()) {+ Some(v) => v as u32, None => return ToolResult::error("Missing required parameter: pid"),+ };+ let xid = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v, None => return ToolResult::error("window_id is required on Linux."),+ };++ let coerce = |key: &str| -> Option<f64> {+ args.get(key).and_then(|v| v.as_f64())+ .or_else(|| args.get(key).and_then(|v| v.as_i64()).map(|i| i as f64))+ };+ let mut from_x = match coerce("from_x") { Some(v) => v, None => return ToolResult::error("Missing: from_x") };+ let mut from_y = match coerce("from_y") { Some(v) => v, None => return ToolResult::error("Missing: from_y") };+ let mut to_x = match coerce("to_x") { Some(v) => v, None => return ToolResult::error("Missing: to_x") };+ let mut to_y = match coerce("to_y") { Some(v) => v, None => return ToolResult::error("Missing: to_y") };++ let duration_ms = args.get("duration_ms").and_then(|v| v.as_u64()).unwrap_or(500);+ let steps = args.get("steps").and_then(|v| v.as_u64()).unwrap_or(20) as usize;+ let button_str = args.get("button").and_then(|v| v.as_str()).unwrap_or("left");+ let button: u8 = match button_str { "right" => 3, "middle" => 2, _ => 1 };+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);++ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => {+ let (wx, wy) = ctx.zoom_to_window(from_x, from_y);+ let (wx2, wy2) = ctx.zoom_to_window(to_x, to_y);+ from_x = wx; from_y = wy; to_x = wx2; to_y = wy2;+ }+ None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ from_x *= ratio; from_y *= ratio;+ to_x *= ratio; to_y *= ratio;+ }++ let result = tokio::task::spawn_blocking(move || {+ crate::input::send_drag(+ xid,+ from_x as i32, from_y as i32,+ to_x as i32, to_y as i32,+ duration_ms, steps, button,+ )+ }).await;++ match result {+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Posted drag ({button_str}) to pid {pid} \+ from ({from_x:.0}, {from_y:.0}) → ({to_x:.0}, {to_y:.0}) \+ in {duration_ms}ms / {steps} steps."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── get_screen_size ───────────────────────────────────────────────────────────++pub struct GetScreenSizeTool;+static GSS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetScreenSizeTool {+ fn def(&self) -> &ToolDef {+ GSS_DEF.get_or_init(|| ToolDef {+ name: "get_screen_size".into(),+ description: "Return the logical size of the main display in points plus its backing \+ scale factor. Agents click in points; Retina displays have scale_factor 2.0. \+ Requires no TCC permissions.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ let result = tokio::task::spawn_blocking(|| {+ use x11rb::connection::Connection;+ use x11rb::rust_connection::RustConnection;+ let (conn, screen_num) = RustConnection::connect(None)?;+ let setup = conn.setup();+ let screen = &setup.roots[screen_num];+ // X11 reports pixel dimensions; scale factor on X11 is not+ // well-defined per-monitor, so report 1.0 (matches DPI-unaware+ // assumption). Wayland/HiDPI X11 callers should query+ // `xrandr --query` for true scale.+ Ok::<(u32, u32, f64), anyhow::Error>((+ screen.width_in_pixels as u32,+ screen.height_in_pixels as u32,+ 1.0,+ ))+ }).await;+ match result {+ // Matches Swift text format 1:1.+ Ok(Ok((w, h, scale))) => ToolResult::text(format!("✅ Main display: {w}x{h} points @ {scale}x"))+ .with_structured(json!({ "width": w, "height": h, "scale_factor": scale })),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── get_cursor_position ───────────────────────────────────────────────────────++pub struct GetCursorPositionTool;+static GCP_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetCursorPositionTool {+ fn def(&self) -> &ToolDef {+ GCP_DEF.get_or_init(|| ToolDef {+ name: "get_cursor_position".into(),+ description: "Return the current mouse cursor position in screen points (origin top-left).".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ let result = tokio::task::spawn_blocking(|| {+ use x11rb::connection::Connection;+ use x11rb::protocol::xproto::ConnectionExt as _;+ use x11rb::rust_connection::RustConnection;+ let (conn, screen_num) = RustConnection::connect(None)?;+ let root = conn.setup().roots[screen_num].root;+ let reply = conn.query_pointer(root)?.reply()?;+ Ok::<(i32, i32), anyhow::Error>((reply.root_x as i32, reply.root_y as i32))+ }).await;+ match result {+ // Text format matches Swift `GetCursorPositionTool` 1:1.+ Ok(Ok((x, y))) => ToolResult::text(format!("✅ Cursor at ({x}, {y})"))+ .with_structured(json!({ "x": x, "y": y })),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── move_cursor ───────────────────────────────────────────────────────────────++pub struct MoveCursorTool {+ state: Arc<ToolState>,+}++static MCURSOR_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for MoveCursorTool {+ fn def(&self) -> &ToolDef {+ MCURSOR_DEF.get_or_init(|| ToolDef {+ name: "move_cursor".into(),+ description: "Move the agent cursor overlay to (x, y). Does NOT move the real mouse cursor.".into(),+ input_schema: json!({"type":"object","required":["x","y"],"properties":{+ "x":{"type":"number"},"y":{"type":"number"},"cursor_id":{"type":"string"}+ },"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ self.state.cursor_registry.update_position(cursor_id, x, y);+ // End pointing upper-left (45°) — matches Swift's+ // `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention so the+ // overlay arrow settles to the natural macOS-style pose.+ crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo {+ x, y, end_heading_radians: std::f64::consts::FRAC_PI_4,+ });+ ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1})."))+ }+}++// ── set_agent_cursor_enabled ──────────────────────────────────────────────────++pub struct SetAgentCursorEnabledTool {+ state: Arc<ToolState>,+}++static SCE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetAgentCursorEnabledTool {+ fn def(&self) -> &ToolDef {+ SCE_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_enabled".into(),+ description: "Show or hide the agent cursor overlay.".into(),+ input_schema: json!({"type":"object","required":["enabled"],"properties":{+ "enabled":{"type":"boolean"},"cursor_id":{"type":"string"}+ },"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let enabled = match args.get("enabled").and_then(|v| v.as_bool()) {+ Some(v) => v, None => return ToolResult::error("Missing required parameter: enabled"),+ };+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ self.state.cursor_registry.set_enabled(cursor_id, enabled);+ crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled));+ ToolResult::text(format!("Agent cursor '{cursor_id}' {}.", if enabled { "enabled" } else { "disabled" }))+ }+}++// ── set_agent_cursor_motion ───────────────────────────────────────────────────++pub struct SetAgentCursorMotionTool {+ state: Arc<ToolState>,+}++static CURSOR_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetAgentCursorMotionTool {+ fn def(&self) -> &ToolDef {+ CURSOR_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_motion".into(),+ description: "Configure the visual appearance of an agent cursor instance.\n\n\+ - cursor_id: instance name (default='default')\n\+ - cursor_icon: built-in ('arrow','crosshair','hand','dot') or PNG/SVG file path\n\+ - cursor_color: hex color e.g. '#00FFFF' or CSS name\n\+ - cursor_label: short text shown near the cursor\n\+ - cursor_size: dot radius in points (default=16)\n\+ - cursor_opacity: 0.0–1.0 (default=0.85)".into(),+ input_schema: json!({+ "type":"object","properties":{+ "cursor_id":{"type":"string"},+ "cursor_icon":{"type":"string"},+ "cursor_color":{"type":"string"},+ "cursor_label":{"type":"string"},+ "cursor_size":{"type":"number"},+ "cursor_opacity":{"type":"number"}+ },"additionalProperties":false+ }),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned();+ self.state.cursor_registry.update_config(&cursor_id, |cfg| {+ if let Some(v) = args.get("cursor_icon").and_then(|v| v.as_str()) { cfg.cursor_icon = Some(v.to_owned()); }+ if let Some(v) = args.get("cursor_color").and_then(|v| v.as_str()) { cfg.cursor_color = Some(v.to_owned()); }+ if let Some(v) = args.get("cursor_label").and_then(|v| v.as_str()) { cfg.cursor_label = Some(v.to_owned()); }+ if let Some(v) = args.get("cursor_size").and_then(|v| v.as_f64()) { cfg.cursor_size = Some(v); }+ if let Some(v) = args.get("cursor_opacity").and_then(|v| v.as_f64()) { cfg.cursor_opacity = Some(v.clamp(0.0, 1.0)); }+ });+ ToolResult::text(format!("Cursor '{cursor_id}' config updated.")).with_structured(args)+ }+}++// ── get_agent_cursor_state ────────────────────────────────────────────────────++pub struct GetAgentCursorStateTool {+ state: Arc<ToolState>,+}++static GCSTATE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetAgentCursorStateTool {+ fn def(&self) -> &ToolDef {+ GCSTATE_DEF.get_or_init(|| ToolDef {+ name: "get_agent_cursor_state".into(),+ description: "Return the current state of all agent cursor instances.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ let states = self.state.cursor_registry.all_states();+ let json = serde_json::to_value(&states).unwrap_or_default();+ ToolResult::text(format!("{} cursor instance(s).", states.len()))+ .with_structured(json!({ "cursors": json }))+ }+}++// ── set_agent_cursor_style ────────────────────────────────────────────────────++pub struct SetAgentCursorStyleTool {+ state: Arc<ToolState>,+}++static STYLE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetAgentCursorStyleTool {+ fn def(&self) -> &ToolDef {+ STYLE_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_style".into(),+ description:+ "Update the visual style of the agent cursor overlay.\n\n\+ - gradient_colors: array of CSS hex strings (e.g. [\"#FF0000\",\"#0000FF\"]) \+ used as the arrow fill gradient from tip to tail. Empty array reverts to \+ the default palette colours.\n\+ - bloom_color: hex string for the radial halo/bloom behind the cursor \+ (e.g. \"#00FFFF\"). Empty string reverts to the default.\n\+ - image_path: path to a PNG, JPEG, SVG, or ICO file to use as the cursor \+ icon instead of the default gradient arrow. Empty string reverts to the \+ procedural arrow.\n\+ All parameters are optional; omit any you do not want to change."+ .into(),+ input_schema: json!({+ "type": "object",+ "properties": {+ "cursor_id": {+ "type": "string",+ "description": "Cursor instance. Default: 'default'."+ },+ "gradient_colors": {+ "type": "array",+ "items": { "type": "string" },+ "description": "CSS hex gradient stops tip→tail. [] = revert to default."+ },+ "bloom_color": {+ "type": "string",+ "description": "Hex bloom/halo colour (e.g. '#00FFFF'). '' = revert to default."+ },+ "image_path": {+ "type": "string",+ "description": "Path to PNG/JPEG/SVG/ICO cursor image. '' = revert to arrow."+ }+ },+ "additionalProperties": false+ }),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned();++ // image_path+ let image_path = args.get("image_path").and_then(|v| v.as_str());+ let shape_cmd: Option<cursor_overlay::OverlayCommand> = if let Some(path) = image_path {+ if path.is_empty() {+ Some(cursor_overlay::OverlayCommand::SetShape(None))+ } else {+ let path_owned = path.to_owned();+ match tokio::task::spawn_blocking(move || {+ cursor_overlay::CursorShape::load(&path_owned)+ }).await {+ Ok(Ok(shape)) => {+ let path_for_cfg = path.to_owned();+ self.state.cursor_registry.update_config(&cursor_id, |cfg| {+ cfg.cursor_icon = Some(path_for_cfg);+ });+ Some(cursor_overlay::OverlayCommand::SetShape(Some(shape)))+ }+ Ok(Err(e)) => return ToolResult::error(format!("Failed to load image_path: {e}")),+ Err(e) => return ToolResult::error(format!("Task error: {e}")),+ }+ }+ } else {+ None+ };++ // gradient_colors+ let gradient_colors: Vec<[u8; 4]> = if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {+ let mut out = vec![];+ for v in arr {+ if let Some(hex) = v.as_str() {+ match parse_hex_color(hex) {+ Some(c) => out.push(c),+ None => return ToolResult::error(format!("Invalid hex color: {hex}")),+ }+ }+ }+ out+ } else {+ vec![]+ };++ // bloom_color+ let bloom_color: Option<Option<[u8; 4]>> = if let Some(hex) = args.get("bloom_color").and_then(|v| v.as_str()) {+ if hex.is_empty() {+ Some(None)+ } else {+ match parse_hex_color(hex) {+ Some(c) => Some(Some(c)),+ None => return ToolResult::error(format!("Invalid bloom_color: {hex}")),+ }+ }+ } else {+ None+ };++ // Dispatch to overlay+ if let Some(cmd) = shape_cmd {+ crate::overlay::send_command(cmd);+ }+ let gradient_provided = args.get("gradient_colors").is_some();+ let bloom_provided = args.get("bloom_color").is_some();+ if gradient_provided || bloom_provided {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::SetGradient {+ gradient_colors,+ bloom_color: bloom_color.flatten(),+ });+ }++ let grad_str = args.get("gradient_colors")+ .and_then(|v| v.as_array())+ .map(|arr| {+ let strs: Vec<String> = arr.iter()+ .filter_map(|v| v.as_str().map(str::to_owned))+ .collect();+ format!("[{}]", strs.join(", "))+ })+ .unwrap_or_else(|| "(unchanged)".into());+ let bloom_str = args.get("bloom_color")+ .and_then(|v| v.as_str())+ .map(|s| if s.is_empty() { "(reverted)".to_owned() } else { s.to_owned() })+ .unwrap_or_else(|| "(unchanged)".into());+ let img_str = image_path+ .map(|s| if s.is_empty() { "(reverted to arrow)".to_owned() } else { s.to_owned() })+ .unwrap_or_else(|| "(unchanged)".into());++ ToolResult::text(format!(+ "cursor style: gradient_colors={grad_str} bloom_color={bloom_str} image_path={img_str}"+ ))+ }+}++fn parse_hex_color(hex: &str) -> Option<[u8; 4]> {+ let s = hex.trim_start_matches('#');+ match s.len() {+ 6 => {+ let r = u8::from_str_radix(&s[0..2], 16).ok()?;+ let g = u8::from_str_radix(&s[2..4], 16).ok()?;+ let b = u8::from_str_radix(&s[4..6], 16).ok()?;+ Some([r, g, b, 255])+ }+ 3 => {+ let r = u8::from_str_radix(&s[0..1].repeat(2), 16).ok()?;+ let g = u8::from_str_radix(&s[1..2].repeat(2), 16).ok()?;+ let b = u8::from_str_radix(&s[2..3].repeat(2), 16).ok()?;+ Some([r, g, b, 255])+ }+ _ => None,+ }+}++// ── check_permissions ─────────────────────────────────────────────────────────++pub struct CheckPermissionsTool;+static PERMS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for CheckPermissionsTool {+ fn def(&self) -> &ToolDef {+ PERMS_DEF.get_or_init(|| ToolDef {+ name: "check_permissions".into(),+ description: "Check required permissions for cua-driver-rs on Linux.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ // Check X11 connectivity (required for window enumeration and input injection).+ let x11_ok = tokio::task::spawn_blocking(|| {+ x11rb::rust_connection::RustConnection::connect(None).is_ok()+ }).await.unwrap_or(false);++ // Check AT-SPI (required for accessibility tree).+ let atspi_ok = std::env::var("DBUS_SESSION_BUS_ADDRESS").is_ok()+ || std::path::Path::new("/run/user").exists();++ let status_text = format!(+ "X11 display: {}\nAT-SPI (D-Bus): {}\nXSendEvent injection: {}",+ if x11_ok { "✅ connected" } else { "❌ DISPLAY not set or X11 unavailable" },+ if atspi_ok { "✅ D-Bus session available" } else { "⚠️ D-Bus session not detected" },+ if x11_ok { "✅ available" } else { "❌ requires X11" }+ );+ ToolResult::text(status_text)+ .with_structured(json!({ "x11": x11_ok, "atspi": atspi_ok, "xsend_event": x11_ok }))+ }+}++// ── get_config ────────────────────────────────────────────────────────────────++pub struct GetConfigTool {+ state: Arc<ToolState>,+}+static GCFG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetConfigTool {+ fn def(&self) -> &ToolDef {+ GCFG_DEF.get_or_init(|| ToolDef {+ name: "get_config".into(),+ description: "Return current cua-driver-rs configuration.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ let cfg = self.state.config.read().unwrap();+ ToolResult::text("cua-driver-rs configuration")+ .with_structured(json!({+ "version": env!("CARGO_PKG_VERSION"),+ "platform": "linux",+ "capture_mode": cfg.capture_mode,+ "max_image_dimension": cfg.max_image_dimension+ }))+ }+}++// ── set_config ────────────────────────────────────────────────────────────────++pub struct SetConfigTool {+ state: Arc<ToolState>,+}+static SCFG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetConfigTool {+ fn def(&self) -> &ToolDef {+ SCFG_DEF.get_or_init(|| ToolDef {+ name: "set_config".into(),+ description: "Update cua-driver-rs configuration. Changes take effect immediately.".into(),+ input_schema: json!({"type":"object","properties":{+ "capture_mode":{"type":"string","enum":["som","vision","ax"],"description":"Default capture mode for get_window_state."},+ "max_image_dimension":{"type":"integer","description":"Max dimension for screenshot resizing (0 = no limit)."}+ },"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let mut cfg = self.state.config.write().unwrap();+ let mut parts = Vec::new();+ if let Some(mode) = args.get("capture_mode").and_then(|v| v.as_str()) {+ cfg.capture_mode = mode.to_owned();+ parts.push(format!("capture_mode={mode}"));+ }+ if let Some(dim) = args.get("max_image_dimension").and_then(|v| v.as_u64()) {+ cfg.max_image_dimension = dim as u32;+ parts.push(format!("max_image_dimension={dim}"));+ }+ let msg = if parts.is_empty() {+ "Config unchanged (no known parameters).".to_owned()+ } else {+ format!("Config updated: {}", parts.join(", "))+ };+ ToolResult::text(msg)+ .with_structured(json!({+ "capture_mode": cfg.capture_mode,+ "max_image_dimension": cfg.max_image_dimension+ }))+ }+}++// ── get_accessibility_tree ────────────────────────────────────────────────────++pub struct GetAccessibilityTreeTool;++static GAX_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetAccessibilityTreeTool {+ fn def(&self) -> &ToolDef {+ GAX_DEF.get_or_init(|| ToolDef {+ name: "get_accessibility_tree".into(),+ description: "Return a lightweight snapshot of the desktop: running processes and \+ on-screen visible X11 windows with their bounds and owner pid.\n\n\+ For the full AT-SPI subtree of a single window (with interactive element indices \+ you can click by), use get_window_state instead — this is a fast discovery read.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ let (procs, windows) = tokio::task::spawn_blocking(|| {+ (crate::proc_fs::list_processes(), crate::x11::list_windows(None))+ }).await.unwrap_or_default();++ let mut lines = vec![format!(+ "{} running process(es), {} visible window(s)",+ procs.len(), windows.len()+ )];+ for p in &procs {+ let cmd = if p.cmdline.is_empty() { p.name.clone() } else { p.cmdline.clone() };+ lines.push(format!("- {} (pid {})", cmd, p.pid));+ }+ if !windows.is_empty() {+ lines.push(String::new());+ lines.push("Windows:".to_owned());+ for w in &windows {+ let title = if w.title.is_empty() { "(no title)".to_owned() }+ else { format!("\"{}\"", w.title) };+ lines.push(format!(+ "- pid={:?} {} [window_id: {}] {}x{}+{}+{}",+ w.pid, title, w.xid, w.width, w.height, w.x, w.y+ ));+ }+ lines.push("→ Call get_window_state(pid, window_id) to inspect a window's UI.".to_owned());+ }++ let structured = json!({+ "processes": procs.iter().map(|p| json!({"pid":p.pid,"name":p.name})).collect::<Vec<_>>(),+ "windows": windows.iter().map(|w| json!({+ "window_id": w.xid, "pid": w.pid, "title": w.title,+ "x": w.x, "y": w.y, "width": w.width, "height": w.height+ })).collect::<Vec<_>>()+ });+ ToolResult::text(lines.join("\n")).with_structured(structured)+ }+}++// ── zoom ──────────────────────────────────────────────────────────────────────++pub struct ZoomTool {+ state: Arc<ToolState>,+}+static ZOOM_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ZoomTool {+ fn def(&self) -> &ToolDef {+ ZOOM_DEF.get_or_init(|| ToolDef {+ name: "zoom".into(),+ description: "Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in \+ screenshot pixels, with 20% padding. Output is at most 500 px wide.\n\n\+ After a zoom, pass from_zoom=true to click/type_text to auto-translate \+ coordinates back to full-window space.".into(),+ input_schema: json!({+ "type":"object","required":["window_id","x1","y1","x2","y2"],"properties":{+ "window_id":{"type":"integer"},+ "pid":{"type":"integer","description":"Target pid — required for from_zoom click/type translation."},+ "x1":{"type":"number"},"y1":{"type":"number"},+ "x2":{"type":"number"},"y2":{"type":"number"}+ },"additionalProperties":false+ }),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let xid = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v, None => return ToolResult::error("Missing required parameter: window_id"),+ };+ let pid = args.get("pid").and_then(|v| v.as_u64()).map(|v| v as u32);+ let x1 = match args.get("x1").and_then(|v| v.as_f64()) { Some(v) => v, None => return ToolResult::error("Missing x1") };+ let y1 = match args.get("y1").and_then(|v| v.as_f64()) { Some(v) => v, None => return ToolResult::error("Missing y1") };+ let x2 = match args.get("x2").and_then(|v| v.as_f64()) { Some(v) => v, None => return ToolResult::error("Missing x2") };+ let y2 = match args.get("y2").and_then(|v| v.as_f64()) { Some(v) => v, None => return ToolResult::error("Missing y2") };+ if x2 <= x1 || y2 <= y1 { return ToolResult::error("x2 must be > x1 and y2 must be > y1"); }++ let state = self.state.clone();+ let result = tokio::task::spawn_blocking(move || {+ let png = crate::capture::screenshot_window_bytes(xid)?;+ cursor_overlay::capture_utils::crop_png_to_jpeg(&png, x1, y1, x2, y2, 500)+ }).await;++ match result {+ Ok(Ok(crop)) => {+ if let Some(p) = pid {+ state.zoom_registry.set(p, ZoomContext {+ origin_x: crop.origin_x,+ origin_y: crop.origin_y,+ scale_inv: crop.scale_inv,+ });+ }+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};+ let b64 = B64.encode(&crop.jpeg_bytes);+ let (w, h) = (crop.out_w, crop.out_h);+ use mcp_server::protocol::Content;+ ToolResult {+ content: vec![+ Content::image_jpeg(b64),+ Content::text(format!("Zoom ({x1:.0},{y1:.0})–({x2:.0},{y2:.0}) → {w}×{h} px JPEG.")),+ ],+ is_error: None,+ structured_content: Some(json!({ "width": w, "height": h, "format": "jpeg" })),+ }+ }+ Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── type_text_chars ───────────────────────────────────────────────────────────++pub struct TypeTextCharsTool;+static TYPE_CHARS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for TypeTextCharsTool {+ fn def(&self) -> &ToolDef {+ TYPE_CHARS_DEF.get_or_init(|| ToolDef {+ name: "type_text_chars".into(),+ description: "Type text character-by-character with a configurable inter-character \+ delay (default 30 ms). Useful for apps that miss keystrokes. \+ Otherwise identical to type_text (XSendEvent, no focus steal).".into(),+ input_schema: json!({+ "type":"object","required":["pid","text"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "text":{"type":"string"},+ "delay_ms":{"type":"integer","description":"Milliseconds between chars (default 30)."},+ "element_index":{"type":"integer"},+ "type_chars_only":{"type":"boolean","description":"Skip element focus, type directly. Default false."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32;+ let text = match args.get("text").and_then(|v| v.as_str()) {+ Some(t) => t.to_owned(), None => return ToolResult::error("Missing required parameter: text"),+ };+ let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30);+ let xid_opt = args.get("window_id").and_then(|v| v.as_u64());+ let xid = match xid_opt {+ Some(x) => x,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.xid,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };+ let text_len = text.chars().count();+ let result = tokio::task::spawn_blocking(move || {+ crate::input::send_type_text_with_delay(xid, &text, delay_ms)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) with {delay_ms}ms delay.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── registry ─────────────────────────────────────────────────────────────────++pub fn build_registry() -> ToolRegistry {+ let state = ToolState::new();+ let mut r = ToolRegistry::new();+ r.register(Box::new(ListAppsTool));+ r.register(Box::new(ListWindowsTool));+ r.register(Box::new(GetWindowStateTool { state: state.clone() }));+ r.register(Box::new(LaunchAppTool));+ r.register(Box::new(ClickTool { state: state.clone() }));+ r.register(Box::new(DoubleClickTool { state: state.clone() }));+ r.register(Box::new(RightClickTool { state: state.clone() }));+ r.register(Box::new(DragTool { state: state.clone() }));+ r.register(Box::new(TypeTextTool));+ r.register(Box::new(PressKeyTool));+ r.register(Box::new(HotkeyTool));+ r.register(Box::new(SetValueTool));+ r.register(Box::new(ScrollTool));+ r.register(Box::new(ScreenshotTool { state: state.clone() }));+ r.register(Box::new(GetScreenSizeTool));+ r.register(Box::new(GetCursorPositionTool));+ r.register(Box::new(MoveCursorTool { state: state.clone() }));+ r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() }));+ r.register(Box::new(SetAgentCursorMotionTool { state: state.clone() }));+ r.register(Box::new(GetAgentCursorStateTool { state: state.clone() }));+ r.register(Box::new(SetAgentCursorStyleTool { state: state.clone() }));+ r.register(Box::new(CheckPermissionsTool));+ r.register(Box::new(GetConfigTool { state: state.clone() }));+ r.register(Box::new(SetConfigTool { state: state.clone() }));+ r.register(Box::new(GetAccessibilityTreeTool));+ r.register(Box::new(ZoomTool { state: state.clone() }));+ r.register(Box::new(TypeTextCharsTool));+ r.register_recording_tools();+ r+}diff --git a/libs/cua-driver-rs/crates/platform-linux/src/tools/mod.rs b/libs/cua-driver-rs/crates/platform-linux/src/tools/mod.rsnew file mode 100644index 0000000000..f511768c65--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/tools/mod.rs@@ -0,0 +1,23 @@+//! Linux tool implementations.+//!+//! On Linux: delegates to real x11/atspi/input/capture implementations.+//! On other platforms: returns "not implemented" stubs so the crate compiles.++use mcp_server::tool::ToolRegistry;++#[cfg(target_os = "linux")]+mod impl_;++#[cfg(not(target_os = "linux"))]+mod stubs;++pub fn build_registry() -> ToolRegistry {+ #[cfg(target_os = "linux")]+ return impl_::build_registry();++ #[cfg(not(target_os = "linux"))]+ stubs::build_registry()+}++// Keep register_all as alias for backwards compat.+pub fn register_all() -> ToolRegistry { build_registry() }diff --git a/libs/cua-driver-rs/crates/platform-linux/src/tools/stubs.rs b/libs/cua-driver-rs/crates/platform-linux/src/tools/stubs.rsnew file mode 100644index 0000000000..e23761f219--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/tools/stubs.rs@@ -0,0 +1,227 @@+//! Non-Linux stubs — return a helpful "not implemented" message.+//! Schemas match the macOS platform-macos implementations (cross-platform interface).++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};+use serde_json::Value;++fn not_impl(name: &str) -> ToolResult {+ ToolResult::error(format!(+ "{name}: Linux implementation is only available when compiled for Linux (target_os = \"linux\")."+ ))+}++macro_rules! stub_tool {+ ($mod_name:ident, $struct_name:ident, $tool_name:literal, $desc:literal, $schema:expr) => {+ mod $mod_name {+ use super::*;+ pub struct $struct_name;+ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+ impl $struct_name {+ pub fn def_static() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: $tool_name.into(),+ description: $desc.into(),+ input_schema: $schema,+ read_only: false,+ destructive: false,+ idempotent: false,+ open_world: false,+ })+ }+ }+ #[async_trait]+ impl Tool for $struct_name {+ fn def(&self) -> &ToolDef { Self::def_static() }+ async fn invoke(&self, _args: Value) -> ToolResult { not_impl($tool_name) }+ }+ }+ pub use $mod_name::$struct_name;+ };+}++stub_tool!(list_apps_m, ListAppsTool, "list_apps",+ "List running regular-UI applications.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(list_windows_m, ListWindowsTool, "list_windows",+ "List top-level windows, optionally filtered by pid.",+ serde_json::json!({"type":"object","properties":{"pid":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(get_window_state_m, GetWindowStateTool, "get_window_state",+ "Walk a running app's AT-SPI tree and return a Markdown rendering of its UI. Also captures a screenshot.",+ serde_json::json!({"type":"object","required":["pid","window_id"],"properties":{"pid":{"type":"integer"},"window_id":{"type":"integer"},"query":{"type":"string"},"capture_mode":{"type":"string","enum":["som","vision","ax"]}},"additionalProperties":false}));++stub_tool!(launch_app_m, LaunchAppTool, "launch_app",+ "Launch an app in the background. Provide name or bundle_id.",+ serde_json::json!({"type":"object","properties":{"bundle_id":{"type":"string"},"name":{"type":"string"},"urls":{"type":"array","items":{"type":"string"}}},"additionalProperties":false}));++stub_tool!(click_m, ClickTool, "click",+ "Click at (x, y) or on an AT-SPI element by element_index + window_id.",+ serde_json::json!({"type":"object","required":["pid"],"properties":{"pid":{"type":"integer"},"window_id":{"type":"integer"},"element_index":{"type":"integer"},"x":{"type":"number"},"y":{"type":"number"},"modifier":{"type":"array","items":{"type":"string"}},"from_zoom":{"type":"boolean"}},"additionalProperties":false}));++stub_tool!(double_click_m, DoubleClickTool, "double_click",+ "Double-click at (x, y) or on an AT-SPI element by element_index + window_id.",+ serde_json::json!({"type":"object","required":["pid"],"properties":{"pid":{"type":"integer"},"x":{"type":"number"},"y":{"type":"number"},"window_id":{"type":"integer"},"element_index":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(right_click_m, RightClickTool, "right_click",+ "Right-click (AT-SPI action on element, or synthesized event at x,y).",+ serde_json::json!({"type":"object","required":["pid"],"properties":{"pid":{"type":"integer"},"element_index":{"type":"integer"},"window_id":{"type":"integer"},"x":{"type":"number"},"y":{"type":"number"},"modifier":{"type":"array","items":{"type":"string"}}},"additionalProperties":false}));++stub_tool!(type_text_m, TypeTextTool, "type_text",+ "Insert text into the target pid via AT-SPI SetTextContents or XSendEvent keystrokes.",+ serde_json::json!({"type":"object","required":["pid","text"],"properties":{"pid":{"type":"integer"},"text":{"type":"string"},"element_index":{"type":"integer"},"window_id":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(type_chars_m, TypeTextCharsTool, "type_text_chars",+ "Type text character-by-character with configurable per-character delay.",+ serde_json::json!({"type":"object","required":["pid","text"],"properties":{"pid":{"type":"integer"},"text":{"type":"string"},"delay_ms":{"type":"integer"},"window_id":{"type":"integer"},"element_index":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(press_key_m, PressKeyTool, "press_key",+ "Press and release a single key delivered directly to the target pid. No focus steal.",+ serde_json::json!({"type":"object","required":["pid","key"],"properties":{"pid":{"type":"integer"},"key":{"type":"string"},"modifiers":{"type":"array","items":{"type":"string"}},"window_id":{"type":"integer"},"element_index":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(hotkey_m, HotkeyTool, "hotkey",+ "Press a combination of keys simultaneously, e.g. [\"ctrl\",\"c\"] for Copy.",+ serde_json::json!({"type":"object","required":["pid","keys"],"properties":{"pid":{"type":"integer"},"keys":{"type":"array","items":{"type":"string"},"minItems":2}},"additionalProperties":false}));++stub_tool!(set_value_m, SetValueTool, "set_value",+ "Set the value of an AT-SPI element (text field, dropdown, checkbox).",+ serde_json::json!({"type":"object","required":["pid","window_id","element_index","value"],"properties":{"pid":{"type":"integer"},"window_id":{"type":"integer"},"element_index":{"type":"integer"},"value":{"type":"string"}},"additionalProperties":false}));++stub_tool!(scroll_m, ScrollTool, "scroll",+ "Scroll the target pid's focused region. direction required; by defaults to line, amount defaults to 3.",+ serde_json::json!({"type":"object","required":["pid","direction"],"properties":{"pid":{"type":"integer"},"direction":{"type":"string","enum":["up","down","left","right"]},"by":{"type":"string","enum":["line","page"]},"amount":{"type":"integer","minimum":1,"maximum":50},"window_id":{"type":"integer"},"element_index":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(screenshot_m, ScreenshotTool, "screenshot",+ "Capture a screenshot. Without window_id captures the full display. Supports png and jpeg formats.",+ serde_json::json!({"type":"object","properties":{"window_id":{"type":"integer"},"format":{"type":"string","enum":["png","jpeg"]},"quality":{"type":"integer","minimum":1,"maximum":95}},"additionalProperties":false}));++stub_tool!(get_screen_size_m, GetScreenSizeTool, "get_screen_size",+ "Return the main screen's width and height in points.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(get_cursor_position_m, GetCursorPositionTool, "get_cursor_position",+ "Return the current real mouse cursor position.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(cursor_motion_m, SetAgentCursorMotionTool, "set_agent_cursor_motion",+ "Configure the visual appearance of an agent cursor instance.\n\nExtended cursor customization for multi-cursor use cases:\n- cursor_id: instance name (default='default')\n- cursor_icon: built-in ('arrow','crosshair','hand','dot') or PNG/SVG file path\n- cursor_color: hex color e.g. '#00FFFF' or CSS name\n- cursor_label: short text shown near the cursor\n- cursor_size: dot radius in points (default=16)\n- cursor_opacity: 0.0–1.0 (default=0.85)",+ serde_json::json!({"type":"object","properties":{"cursor_id":{"type":"string"},"cursor_icon":{"type":"string"},"cursor_color":{"type":"string"},"cursor_label":{"type":"string"},"cursor_size":{"type":"number"},"cursor_opacity":{"type":"number"}},"additionalProperties":false}));++stub_tool!(get_cursor_state_m, GetAgentCursorStateTool, "get_agent_cursor_state",+ "Return the current state of all agent cursor instances.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(set_cursor_style_m, SetAgentCursorStyleTool, "set_agent_cursor_style",+ "Update the visual style of the agent cursor overlay.",+ serde_json::json!({"type":"object","properties":{"cursor_id":{"type":"string"},"gradient_colors":{"type":"array","items":{"type":"string"}},"bloom_color":{"type":"string"},"image_path":{"type":"string"}},"additionalProperties":false}));++stub_tool!(check_perms_m, CheckPermissionsTool, "check_permissions",+ "Check required permissions.",+ serde_json::json!({"type":"object","properties":{"prompt":{"type":"boolean"}},"additionalProperties":false}));++stub_tool!(get_config_m, GetConfigTool, "get_config",+ "Return current cua-driver-rs configuration.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(set_config_m, SetConfigTool, "set_config",+ "Update cua-driver-rs configuration.",+ serde_json::json!({"type":"object","properties":{"capture_mode":{"type":"string","enum":["som","vision","ax"]},"max_image_dimension":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(get_ax_tree_m, GetAccessibilityTreeTool, "get_accessibility_tree",+ "Lightweight desktop snapshot: running apps and visible windows. For per-window AT-SPI tree, use get_window_state.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(zoom_m, ZoomTool, "zoom",+ "Crop a window region to JPEG with 20% padding, max 500px wide.",+ serde_json::json!({"type":"object","required":["window_id","x1","y1","x2","y2"],"properties":{"window_id":{"type":"integer"},"x1":{"type":"number"},"y1":{"type":"number"},"x2":{"type":"number"},"y2":{"type":"number"},"pid":{"type":"integer"}},"additionalProperties":false}));++// move_cursor and set_agent_cursor_enabled route through the overlay channel.++mod move_cursor_m {+ use super::*;+ pub struct MoveCursorTool;+ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+ impl MoveCursorTool {+ pub fn def_static() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "move_cursor".into(),+ description: "Move the agent cursor overlay to (x, y). Does NOT move the real \+ mouse cursor — the user's cursor stays where it is.".into(),+ input_schema: serde_json::json!({"type":"object","required":["x","y"],"properties":{"x":{"type":"number"},"y":{"type":"number"},"cursor_id":{"type":"string"}},"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ }+ #[async_trait]+ impl Tool for MoveCursorTool {+ fn def(&self) -> &ToolDef { Self::def_static() }+ async fn invoke(&self, args: Value) -> ToolResult {+ let x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo { x, y, end_heading_radians: 0.0 });+ ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1})."))+ }+ }+}+pub use move_cursor_m::MoveCursorTool;++mod set_enabled_m {+ use super::*;+ pub struct SetAgentCursorEnabledTool;+ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+ impl SetAgentCursorEnabledTool {+ pub fn def_static() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_enabled".into(),+ description: "Show or hide the agent cursor overlay.".into(),+ input_schema: serde_json::json!({"type":"object","required":["enabled"],"properties":{"enabled":{"type":"boolean"},"cursor_id":{"type":"string"}},"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ }+ #[async_trait]+ impl Tool for SetAgentCursorEnabledTool {+ fn def(&self) -> &ToolDef { Self::def_static() }+ async fn invoke(&self, args: Value) -> ToolResult {+ let enabled = args.get("enabled").and_then(|v| v.as_bool()).unwrap_or(true);+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled));+ ToolResult::text(format!("Agent cursor '{}' {}.", cursor_id, if enabled { "enabled" } else { "disabled" }))+ }+ }+}+pub use set_enabled_m::SetAgentCursorEnabledTool;++pub fn build_registry() -> mcp_server::tool::ToolRegistry {+ let mut r = mcp_server::tool::ToolRegistry::new();+ r.register(Box::new(ListAppsTool));+ r.register(Box::new(ListWindowsTool));+ r.register(Box::new(GetWindowStateTool));+ r.register(Box::new(LaunchAppTool));+ r.register(Box::new(ClickTool));+ r.register(Box::new(DoubleClickTool));+ r.register(Box::new(RightClickTool));+ r.register(Box::new(TypeTextTool));+ r.register(Box::new(PressKeyTool));+ r.register(Box::new(HotkeyTool));+ r.register(Box::new(SetValueTool));+ r.register(Box::new(ScrollTool));+ r.register(Box::new(ScreenshotTool));+ r.register(Box::new(GetScreenSizeTool));+ r.register(Box::new(GetCursorPositionTool));+ r.register(Box::new(MoveCursorTool));+ r.register(Box::new(SetAgentCursorEnabledTool));+ r.register(Box::new(SetAgentCursorMotionTool));+ r.register(Box::new(GetAgentCursorStateTool));+ r.register(Box::new(SetAgentCursorStyleTool));+ r.register(Box::new(CheckPermissionsTool));+ r.register(Box::new(GetConfigTool));+ r.register(Box::new(SetConfigTool));+ r.register(Box::new(GetAccessibilityTreeTool));+ r.register(Box::new(ZoomTool));+ r.register(Box::new(TypeTextCharsTool));+ r.register_recording_tools();+ r+}diff --git a/libs/cua-driver-rs/crates/platform-linux/src/x11/mod.rs b/libs/cua-driver-rs/crates/platform-linux/src/x11/mod.rsnew file mode 100644index 0000000000..4539f40e09--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-linux/src/x11/mod.rs@@ -0,0 +1,108 @@+//! X11 window enumeration via x11rb.+//!+//! Uses _NET_CLIENT_LIST_STACKING to get the list of top-level windows,+//! then reads WM_NAME/_NET_WM_NAME, _NET_WM_PID, and geometry per window.++use anyhow::Result;+use x11rb::connection::Connection;+use x11rb::protocol::xproto::*;+use x11rb::rust_connection::RustConnection;++#[derive(Debug, Clone)]+pub struct WindowInfo {+ /// X11 Window (XID) cast to u64.+ pub xid: u64,+ pub pid: Option<u32>,+ pub title: String,+ pub x: i32,+ pub y: i32,+ pub width: u32,+ pub height: u32,+}++/// List top-level windows, optionally filtered by pid.+pub fn list_windows(filter_pid: Option<u32>) -> Vec<WindowInfo> {+ match list_windows_inner(filter_pid) {+ Ok(w) => w,+ Err(_) => Vec::new(),+ }+}++fn list_windows_inner(filter_pid: Option<u32>) -> Result<Vec<WindowInfo>> {+ let (conn, screen_num) = RustConnection::connect(None)?;+ let screen = &conn.setup().roots[screen_num];+ let root = screen.root;++ // Get _NET_CLIENT_LIST_STACKING (or fallback to _NET_CLIENT_LIST).+ let windows = get_window_list(&conn, root)?;++ let mut result = Vec::new();+ for xid in windows {+ let pid = get_window_pid(&conn, xid).ok().flatten();+ if let Some(fp) = filter_pid {+ if pid != Some(fp) { continue; }+ }++ let title = get_window_title(&conn, xid).unwrap_or_default();+ if title.trim().is_empty() { continue; }++ let geom = conn.get_geometry(xid)?.reply().ok();+ let (x, y, w, h) = if let Some(g) = geom {+ // Translate to root coordinates.+ let trans = conn.translate_coordinates(xid, root, 0, 0)?.reply().ok();+ let (rx, ry) = trans.map(|t| (t.dst_x as i32, t.dst_y as i32)).unwrap_or((0, 0));+ (rx, ry, g.width as u32, g.height as u32)+ } else {+ (0, 0, 0, 0)+ };++ result.push(WindowInfo { xid: xid as u64, pid, title, x, y, width: w, height: h });+ }++ Ok(result)+}++fn get_window_list(conn: &RustConnection, root: Window) -> Result<Vec<Window>> {+ let atom_names = ["_NET_CLIENT_LIST_STACKING", "_NET_CLIENT_LIST"];+ for name in &atom_names {+ if let Ok(atom) = get_atom(conn, name) {+ if let Ok(reply) = conn.get_property(false, root, atom, AtomEnum::WINDOW, 0, u32::MAX)?.reply() {+ let windows: Vec<Window> = reply.value32()+ .map(|iter| iter.collect())+ .unwrap_or_default();+ if !windows.is_empty() {+ return Ok(windows);+ }+ }+ }+ }+ // Fallback: query tree from root.+ let tree = conn.query_tree(root)?.reply()?;+ Ok(tree.children)+}++fn get_atom(conn: &RustConnection, name: &str) -> Result<Atom> {+ Ok(conn.intern_atom(false, name.as_bytes())?.reply()?.atom)+}++fn get_window_pid(conn: &RustConnection, window: Window) -> Result<Option<u32>> {+ let atom = get_atom(conn, "_NET_WM_PID")?;+ let reply = conn.get_property(false, window, atom, AtomEnum::CARDINAL, 0, 1)?.reply()?;+ Ok(reply.value32().and_then(|mut i| i.next()))+}++fn get_window_title(conn: &RustConnection, window: Window) -> Result<String> {+ // Try _NET_WM_NAME (UTF-8) first.+ if let Ok(atom) = get_atom(conn, "_NET_WM_NAME") {+ if let Ok(utf8_atom) = get_atom(conn, "UTF8_STRING") {+ if let Ok(reply) = conn.get_property(false, window, atom, utf8_atom, 0, 1024)?.reply() {+ if !reply.value.is_empty() {+ return Ok(String::from_utf8_lossy(&reply.value).into_owned());+ }+ }+ }+ }+ // Fallback: WM_NAME (latin-1 / ASCII).+ let reply = conn.get_property(false, window, AtomEnum::WM_NAME, AtomEnum::STRING, 0, 1024)?.reply()?;+ Ok(String::from_utf8_lossy(&reply.value).into_owned())+}diff --git a/libs/cua-driver-rs/crates/platform-macos/Cargo.toml b/libs/cua-driver-rs/crates/platform-macos/Cargo.tomlnew file mode 100644index 0000000000..da75e504f8--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/Cargo.toml@@ -0,0 +1,45 @@+[package]+name = "platform-macos"+version.workspace = true+edition.workspace = true++[dependencies]+tokio = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+anyhow = { workspace = true }+thiserror = { workspace = true }+tracing = { workspace = true }+base64 = { workspace = true }+image = { workspace = true }+async-trait = "0.1"+mcp-server = { path = "../mcp-server" }+cursor-overlay = { path = "../cursor-overlay" }++[target.'cfg(target_os = "macos")'.dependencies]+# macOS system bindings+core-foundation = "0.10"+core-graphics = { version = "0.24", features = ["highsierra", "elcapitan"] }+foreign-types = "0.5"+libc = "0.2"+# objc2 and its AppKit/CoreFoundation bindings+objc2 = "0.5"+objc2-foundation = { version = "0.2", features = [+ "NSArray", "NSString", "NSDictionary", "NSRunLoop",+ "NSThread", "NSObject", "NSValue",+] }+objc2-app-kit = { version = "0.2", features = [+ "NSApplication", "NSWindow", "NSScreen", "NSColor",+ "NSEvent", "NSRunningApplication", "NSWorkspace",+ "NSView", "NSGraphicsContext", "NSResponder",+] }++# QuartzCore for CALayer+objc2-quartz-core = { version = "0.2", features = ["CALayer"] }++# tiny-skia for cursor rendering+tiny-skia = { version = "0.11", default-features = false, features = ["std"] }++# WebSocket CDP client (used by browser/cdp_client.rs for the page tool)+tokio-tungstenite = { version = "0.24", features = ["native-tls"] }+futures-util = "0.3"diff --git a/libs/cua-driver-rs/crates/platform-macos/build.rs b/libs/cua-driver-rs/crates/platform-macos/build.rsnew file mode 100644index 0000000000..7be8898b68--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/build.rs@@ -0,0 +1,34 @@+fn main() {+ let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();+ if target_os != "macos" {+ return;+ }+ // Force AppKit, Foundation, QuartzCore and CoreGraphics to be linked.+ // objc2-app-kit's empty extern block is not enough to propagate+ // framework links through Cargo's dependency graph on all toolchain+ // versions, so we emit the directives explicitly here.+ println!("cargo:rustc-link-lib=framework=AppKit");+ println!("cargo:rustc-link-lib=framework=Foundation");+ println!("cargo:rustc-link-lib=framework=QuartzCore");+ println!("cargo:rustc-link-lib=framework=CoreGraphics");++ // Point the linker at the SDK's system sub-library directory so+ // `#[link(name = "dispatch", kind = "dylib")]` can find libdispatch.tbd.+ // The path is stable across Xcode / CLT installs; SDKROOT takes precedence+ // if set (Xcode builds set it automatically).+ let sdk_root = std::env::var("SDKROOT").unwrap_or_else(|_| {+ // Fall back to the active CLT SDK.+ let out = std::process::Command::new("xcrun")+ .args(["--sdk", "macosx", "--show-sdk-path"])+ .output()+ .ok()+ .and_then(|o| String::from_utf8(o.stdout).ok())+ .unwrap_or_default();+ out.trim().to_owned()+ });++ if !sdk_root.is_empty() {+ println!("cargo:rustc-link-search={sdk_root}/usr/lib/system");+ println!("cargo:rustc-link-search={sdk_root}/usr/lib");+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/apps.rs b/libs/cua-driver-rs/crates/platform-macos/src/apps.rsnew file mode 100644index 0000000000..3c1b4cfc59--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/apps.rs@@ -0,0 +1,249 @@+//! macOS app enumeration via NSWorkspace and NSRunningApplication.++use std::process::Command;+use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct AppInfo {+ pub name: String,+ pub pid: i32,+ pub bundle_id: Option<String>,+ pub running: bool,+ pub active: bool,+}++/// Enumerate running apps with NSApplicationActivationPolicyRegular.+/// Uses `osascript` to query System Events — no Accessibility permission needed.+pub fn list_running_apps() -> Vec<AppInfo> {+ // Use `ps` + NSWorkspace via a brief Swift one-liner via swift-sh is heavy;+ // instead use the Obj-C bridge via `osascript`.+ // Fallback: parse `ps -axo pid,command` and cross-reference with bundle IDs.+ // Better: use a native call via core-foundation bindings.+ list_running_apps_native()+}++fn list_running_apps_native() -> Vec<AppInfo> {+ // Use `lsappinfo list` which is a macOS system tool available on all versions.+ // Alternatively we can use NSWorkspace via objc2 — but for simplicity and+ // to match the Swift reference, we use a subprocess call to `osascript`.+ let script = r#"+set output to ""+tell application "System Events"+ set appList to every application process whose background only is false+ repeat with proc in appList+ set procName to name of proc+ set procPid to unix id of proc+ set bundleId to bundle identifier of proc+ if bundleId is missing value then set bundleId to ""+ set isFront to "0"+ if frontmost of proc then set isFront to "1"+ set output to output & procName & "|" & procPid & "|" & bundleId & "|" & isFront & linefeed+ end repeat+end tell+return output+"#;++ let out = Command::new("osascript")+ .arg("-e")+ .arg(script)+ .output();++ match out {+ Err(_) => vec![],+ Ok(o) => {+ let text = String::from_utf8_lossy(&o.stdout);+ parse_osascript_app_list(&text)+ }+ }+}++fn parse_osascript_app_list(text: &str) -> Vec<AppInfo> {+ let mut apps = Vec::new();+ for line in text.lines() {+ let parts: Vec<&str> = line.splitn(4, '|').collect();+ if parts.len() < 2 { continue; }+ let name = parts[0].trim().to_owned();+ let pid: i32 = parts[1].trim().parse().unwrap_or(0);+ let bundle_id = if parts.len() > 2 && !parts[2].trim().is_empty() {+ Some(parts[2].trim().to_owned())+ } else {+ None+ };+ let active = parts.get(3).map(|s| s.trim() == "1").unwrap_or(false);+ if name.is_empty() || pid == 0 { continue; }+ apps.push(AppInfo { name, pid, bundle_id, running: true, active });+ }+ apps+}++/// Launch an app by bundle ID using `open -g -b` (background, no activation).+/// Returns the pid on success.+pub fn launch_app(bundle_id: &str) -> anyhow::Result<i32> {+ let status = Command::new("open")+ .args(["-g", "-b", bundle_id])+ .status()?;+ if !status.success() {+ anyhow::bail!("Failed to launch {bundle_id}");+ }+ // Give the app a moment to start.+ std::thread::sleep(std::time::Duration::from_millis(500));+ // Find its pid.+ let apps = list_running_apps();+ for app in &apps {+ if app.bundle_id.as_deref() == Some(bundle_id) {+ return Ok(app.pid);+ }+ }+ anyhow::bail!("Launched {bundle_id} but could not find its pid")+}++/// Launch an app by display name using `open -g -a AppName` (background, no activation).+/// Returns the pid on success.+pub fn launch_app_by_name(name: &str) -> anyhow::Result<i32> {+ let status = Command::new("open")+ .args(["-g", "-a", name])+ .status()?;+ if !status.success() {+ anyhow::bail!("Failed to launch app '{name}'");+ }+ std::thread::sleep(std::time::Duration::from_millis(500));+ let apps = list_running_apps();+ for app in &apps {+ if app.name.eq_ignore_ascii_case(name) {+ return Ok(app.pid);+ }+ }+ anyhow::bail!("Launched '{name}' but could not find its pid")+}++/// Return all apps: running apps merged with installed-but-not-running apps.+pub fn list_all_apps() -> Vec<AppInfo> {+ let running = list_running_apps();+ let running_bundles: std::collections::HashSet<String> = running.iter()+ .filter_map(|a| a.bundle_id.clone())+ .collect();++ let mut installed = scan_installed_apps();+ // Remove apps already in running list.+ installed.retain(|a| !a.bundle_id.as_ref().map_or(false, |b| running_bundles.contains(b)));++ let mut all = running;+ all.extend(installed);+ all+}++fn scan_installed_apps() -> Vec<AppInfo> {+ let dirs = [+ "/Applications",+ "/Applications/Utilities",+ "/System/Applications",+ "/System/Applications/Utilities",+ ];+ let home = std::env::var("HOME").unwrap_or_default();+ let user_apps = format!("{home}/Applications");++ let mut result = Vec::new();+ let mut all_dirs: Vec<&str> = dirs.to_vec();+ let user_apps_str: &str = user_apps.as_str();+ all_dirs.push(user_apps_str);++ for dir in all_dirs {+ let Ok(entries) = std::fs::read_dir(dir) else { continue };+ for entry in entries.flatten() {+ let path = entry.path();+ if path.extension().and_then(|e| e.to_str()) != Some("app") { continue }+ let plist_path = path.join("Contents/Info.plist");+ if let Some(info) = read_app_plist(&plist_path) {+ result.push(info);+ }+ }+ }+ result+}++fn read_app_plist(plist_path: &std::path::Path) -> Option<AppInfo> {+ let bundle_id_out = Command::new("plutil")+ .args(["-extract", "CFBundleIdentifier", "raw", "-o", "-",+ plist_path.to_str()?])+ .output().ok()?;+ if !bundle_id_out.status.success() { return None; }+ let bundle_id = String::from_utf8_lossy(&bundle_id_out.stdout).trim().to_string();+ if bundle_id.is_empty() { return None; }++ let name_out = Command::new("plutil")+ .args(["-extract", "CFBundleDisplayName", "raw", "-o", "-",+ plist_path.to_str()?])+ .output().ok();+ let name = name_out+ .filter(|o| o.status.success())+ .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())+ .filter(|s| !s.is_empty())+ .unwrap_or_else(|| {+ // Fallback: CFBundleName.+ Command::new("plutil")+ .args(["-extract", "CFBundleName", "raw", "-o", "-",+ plist_path.to_str().unwrap_or("")])+ .output().ok()+ .filter(|o| o.status.success())+ .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())+ .filter(|s| !s.is_empty())+ .unwrap_or_else(|| {+ plist_path.parent()+ .and_then(|p| p.parent())+ .and_then(|p| p.file_stem())+ .and_then(|s| s.to_str())+ .unwrap_or("")+ .to_string()+ })+ });++ if name.is_empty() { return None; }++ Some(AppInfo {+ name,+ pid: 0,+ bundle_id: Some(bundle_id),+ running: false,+ active: false,+ })+}++/// Return the localized application name for a running process by PID.+/// Uses `ps -p {pid} -o comm=` which gives the command name without path.+/// Returns `None` if the PID is unknown or the command fails.+pub fn get_app_name_for_pid(pid: i32) -> Option<String> {+ let out = Command::new("ps")+ .args(["-p", &pid.to_string(), "-o", "comm="])+ .output()+ .ok()?;+ let raw = String::from_utf8_lossy(&out.stdout).trim().to_string();+ if raw.is_empty() {+ return None;+ }+ // Strip path prefix: "/Applications/Safari.app/Contents/MacOS/Safari" → "Safari"+ Some(+ std::path::Path::new(&raw)+ .file_name()+ .and_then(|n| n.to_str())+ .unwrap_or(&raw)+ .to_string(),+ )+}++/// Format the app list in the same text style as libs/cua-driver.+pub fn format_app_list(apps: &[AppInfo]) -> String {+ let running: Vec<&AppInfo> = apps.iter().filter(|a| a.running).collect();+ let total = apps.len();+ // Match Swift `ListAppsTool.swift` `summary(_:)` text format 1:1.+ let mut lines = vec![format!(+ "✅ Found {} app(s): {} running, {} installed-not-running.",+ total,+ running.len(),+ total - running.len()+ )];+ for app in &running {+ let bundle = app.bundle_id.as_deref().map(|b| format!(" [{b}]")).unwrap_or_default();+ lines.push(format!("- {} (pid {}){}", app.name, app.pid, bundle));+ }+ lines.join("\n")+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver-rs/crates/platform-macos/src/ax/bindings.rsnew file mode 100644index 0000000000..3ad4504364--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/ax/bindings.rs@@ -0,0 +1,326 @@+//! Raw FFI bindings to the macOS Accessibility API (AXUIElement).+//!+//! We call the C-level AX API directly rather than using a crate wrapper,+//! because most available crates are incomplete or unmaintained.++#![allow(non_upper_case_globals, non_camel_case_types, non_snake_case, dead_code)]++use core_foundation::{+ array::CFArrayRef,+ base::{CFIndex, CFRelease, CFRetain, CFTypeID, CFTypeRef},+ string::CFStringRef,+};+use std::os::raw::{c_int, c_void};++// ── AXUIElement opaque type ──────────────────────────────────────────────────++#[repr(C)]+pub struct __AXUIElement(c_void);+pub type AXUIElementRef = *mut __AXUIElement;++// ── AXError ──────────────────────────────────────────────────────────────────++pub type AXError = c_int;+pub const kAXErrorSuccess: AXError = 0;+pub const kAXErrorFailure: AXError = -25200;+pub const kAXErrorInvalidUIElement: AXError = -25202;+pub const kAXErrorAttributeUnsupported: AXError = -25205;+pub const kAXErrorNoValue: AXError = -25212;+pub const kAXErrorAPIDisabled: AXError = -25211;++// ── AXValue opaque type ──────────────────────────────────────────────────────++#[repr(C)]+pub struct __AXValue(c_void);+pub type AXValueRef = *mut __AXValue;++pub type AXValueType = c_int;+pub const kAXValueCGPointType: AXValueType = 1;+pub const kAXValueCGSizeType: AXValueType = 2;+pub const kAXValueCGRectType: AXValueType = 3;+pub const kAXValueCFRangeType: AXValueType = 4;+pub const kAXValueIllegalType: AXValueType = 1_000;++// ── Link to AXUIElement functions ────────────────────────────────────────────+#[link(name = "ApplicationServices", kind = "framework")]+extern "C" {+ pub fn AXUIElementCreateApplication(pid: i32) -> AXUIElementRef;+ pub fn AXUIElementCopyAttributeValue(+ element: AXUIElementRef,+ attribute: CFStringRef,+ value: *mut CFTypeRef,+ ) -> AXError;+ pub fn AXUIElementCopyAttributeNames(+ element: AXUIElementRef,+ names: *mut CFArrayRef,+ ) -> AXError;+ pub fn AXUIElementCopyActionNames(+ element: AXUIElementRef,+ names: *mut CFArrayRef,+ ) -> AXError;+ pub fn AXUIElementPerformAction(+ element: AXUIElementRef,+ action: CFStringRef,+ ) -> AXError;+ pub fn AXUIElementSetAttributeValue(+ element: AXUIElementRef,+ attribute: CFStringRef,+ value: CFTypeRef,+ ) -> AXError;+ pub fn AXUIElementGetTypeID() -> CFTypeID;+ pub fn AXIsProcessTrusted() -> bool;+ /// `AXIsProcessTrustedWithOptions(options)` — when called with+ /// `{kAXTrustedCheckOptionPrompt: true}` raises the system Accessibility+ /// prompt if the process isn't already trusted. Returns the post-prompt+ /// trust state (may still be false if the user dismissed the prompt).+ pub fn AXIsProcessTrustedWithOptions(options: core_foundation::dictionary::CFDictionaryRef) -> bool;++ /// Private SPI: maps an AX window element to its CGWindowID.+ /// Stable since macOS 10.9; used by yabai, Hammerspoon, Accessibility Inspector.+ pub fn _AXUIElementGetWindow(element: AXUIElementRef, window_id: *mut u32) -> AXError;+}++// ── AXValue functions ────────────────────────────────────────────────────────+#[link(name = "ApplicationServices", kind = "framework")]+extern "C" {+ pub fn AXValueGetType(value: AXValueRef) -> AXValueType;+ pub fn AXValueGetValue(value: AXValueRef, the_type: AXValueType, value_ptr: *mut c_void) -> bool;+}++// ── Helper functions ──────────────────────────────────────────────────────────++use core_foundation::{+ array::CFArray,+ base::TCFType,+ string::CFString as CFStr,+};++/// Copy a string attribute from an AX element. Returns `None` on any error.+pub unsafe fn copy_string_attr(element: AXUIElementRef, attr_name: &str) -> Option<String> {+ let attr = CFStr::new(attr_name);+ let mut value: CFTypeRef = std::ptr::null();+ let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value);+ if err != kAXErrorSuccess || value.is_null() {+ return None;+ }+ let cf_string_type_id = CFStr::type_id();+ if core_foundation::base::CFGetTypeID(value) != cf_string_type_id {+ CFRelease(value);+ return None;+ }+ let s = CFStr::wrap_under_create_rule(value as _);+ Some(s.to_string())+}++/// Get the action names for an AX element.+pub unsafe fn copy_action_names(element: AXUIElementRef) -> Vec<String> {+ let mut names: CFArrayRef = std::ptr::null_mut();+ let err = AXUIElementCopyActionNames(element, &mut names);+ if err != kAXErrorSuccess || names.is_null() {+ return vec![];+ }+ // Use CFArray<CFStr> (the typed wrapper) to satisfy FromVoid bound.+ let arr = CFArray::<CFStr>::wrap_under_create_rule(names);+ (0..arr.len())+ .filter_map(|i| {+ let cf = arr.get(i)?;+ Some(cf.to_string())+ })+ .collect()+}++/// Read the on-screen center of an AX element (AXPosition + AXSize → center).+/// Returns `(cx, cy)` in screen coordinates, or `None` if either attribute+/// is unavailable or the element has zero size.+pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64)> {+ // AXPosition → CGPoint+ let pos_attr = CFStr::new("AXPosition");+ let mut pos_ref: CFTypeRef = std::ptr::null();+ let err = AXUIElementCopyAttributeValue(element, pos_attr.as_concrete_TypeRef(), &mut pos_ref);+ if err != kAXErrorSuccess || pos_ref.is_null() {+ return None;+ }+ #[repr(C)]+ struct CGPoint { x: f64, y: f64 }+ let mut pos = CGPoint { x: 0.0, y: 0.0 };+ let ok = AXValueGetValue(+ pos_ref as AXValueRef,+ kAXValueCGPointType,+ &mut pos as *mut _ as *mut std::ffi::c_void,+ );+ CFRelease(pos_ref);+ if !ok { return None; }++ // AXSize → CGSize+ let sz_attr = CFStr::new("AXSize");+ let mut sz_ref: CFTypeRef = std::ptr::null();+ let err2 = AXUIElementCopyAttributeValue(element, sz_attr.as_concrete_TypeRef(), &mut sz_ref);+ if err2 != kAXErrorSuccess || sz_ref.is_null() {+ return None;+ }+ #[repr(C)]+ struct CGSize { w: f64, h: f64 }+ let mut sz = CGSize { w: 0.0, h: 0.0 };+ let ok2 = AXValueGetValue(+ sz_ref as AXValueRef,+ kAXValueCGSizeType,+ &mut sz as *mut _ as *mut std::ffi::c_void,+ );+ CFRelease(sz_ref);+ if !ok2 || sz.w < 1.0 || sz.h < 1.0 { return None; }++ Some((pos.x + sz.w / 2.0, pos.y + sz.h / 2.0))+}++/// Read the on-screen bounding rect of an AX element.+/// Returns `[x, y, width, height]` in screen coordinates (top-left origin), or `None`.+pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> {+ // AXPosition → CGPoint+ let pos_attr = CFStr::new("AXPosition");+ let mut pos_ref: CFTypeRef = std::ptr::null();+ let err = AXUIElementCopyAttributeValue(element, pos_attr.as_concrete_TypeRef(), &mut pos_ref);+ if err != kAXErrorSuccess || pos_ref.is_null() {+ return None;+ }+ #[repr(C)]+ struct CGPoint { x: f64, y: f64 }+ let mut pos = CGPoint { x: 0.0, y: 0.0 };+ let ok = AXValueGetValue(+ pos_ref as AXValueRef,+ kAXValueCGPointType,+ &mut pos as *mut _ as *mut std::ffi::c_void,+ );+ CFRelease(pos_ref);+ if !ok { return None; }++ // AXSize → CGSize+ let sz_attr = CFStr::new("AXSize");+ let mut sz_ref: CFTypeRef = std::ptr::null();+ let err2 = AXUIElementCopyAttributeValue(element, sz_attr.as_concrete_TypeRef(), &mut sz_ref);+ if err2 != kAXErrorSuccess || sz_ref.is_null() {+ return None;+ }+ #[repr(C)]+ struct CGSize { w: f64, h: f64 }+ let mut sz = CGSize { w: 0.0, h: 0.0 };+ let ok2 = AXValueGetValue(+ sz_ref as AXValueRef,+ kAXValueCGSizeType,+ &mut sz as *mut _ as *mut std::ffi::c_void,+ );+ CFRelease(sz_ref);+ if !ok2 || sz.w < 1.0 || sz.h < 1.0 { return None; }++ Some([pos.x, pos.y, sz.w, sz.h])+}++/// Get the focused UI element of a running application by pid.+/// Returns a retained `AXUIElementRef` that the caller must release, or `None`.+pub unsafe fn focused_element_of_pid(pid: i32) -> Option<AXUIElementRef> {+ let app = AXUIElementCreateApplication(pid);+ if app.is_null() {+ return None;+ }+ let attr = CFStr::new("AXFocusedUIElement");+ let mut value: CFTypeRef = std::ptr::null();+ let err = AXUIElementCopyAttributeValue(app, attr.as_concrete_TypeRef(), &mut value);+ CFRelease(app as CFTypeRef);+ if err != kAXErrorSuccess || value.is_null() {+ return None;+ }+ let ax_type_id = AXUIElementGetTypeID();+ if core_foundation::base::CFGetTypeID(value) != ax_type_id {+ CFRelease(value);+ return None;+ }+ // Already retained by CopyAttributeValue — hand the raw pointer to the caller.+ Some(value as AXUIElementRef)+}++/// Get the children of an AX element.+pub unsafe fn copy_children(element: AXUIElementRef) -> Vec<AXUIElementRef> {+ let attr = CFStr::new("AXChildren");+ let mut value: CFTypeRef = std::ptr::null();+ let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value);+ if err != kAXErrorSuccess || value.is_null() {+ return vec![];+ }+ let cf_array_type_id = CFArray::<CFTypeRef>::type_id();+ if core_foundation::base::CFGetTypeID(value) != cf_array_type_id {+ CFRelease(value);+ return vec![];+ }+ let arr = CFArray::<CFTypeRef>::wrap_under_create_rule(value as _);+ let ax_type_id = AXUIElementGetTypeID();+ (0..arr.len())+ .filter_map(|i| {+ let item = *arr.get(i)?;+ if core_foundation::base::CFGetTypeID(item) == ax_type_id {+ // Retain so we own it — caller is responsible for releasing.+ CFRetain(item);+ Some(item as AXUIElementRef)+ } else {+ None+ }+ })+ .collect()+}++/// Perform an AX action using a string attribute name.+pub unsafe fn perform_action(element: AXUIElementRef, action_name: &str) -> AXError {+ let action = CFStr::new(action_name);+ AXUIElementPerformAction(element, action.as_concrete_TypeRef())+}++/// Set an AX attribute to a CFString value.+pub unsafe fn set_string_attr(element: AXUIElementRef, attr_name: &str, value: &str) -> AXError {+ let attr = CFStr::new(attr_name);+ let cf_value = CFStr::new(value);+ AXUIElementSetAttributeValue(element, attr.as_concrete_TypeRef(), cf_value.as_CFTypeRef())+}++/// Set an AX attribute to a CFBoolean true value.+pub unsafe fn set_bool_attr_true(element: AXUIElementRef, attr_name: &str) -> AXError {+ use core_foundation::boolean::CFBoolean;+ let attr = CFStr::new(attr_name);+ let cf_true = CFBoolean::true_value();+ AXUIElementSetAttributeValue(element, attr.as_concrete_TypeRef(), cf_true.as_CFTypeRef())+}++/// Get the CGWindowID of an AX window element via the private `_AXUIElementGetWindow` SPI.+/// Returns `None` if the element is not a composited window.+pub unsafe fn ax_get_window_id(element: AXUIElementRef) -> Option<u32> {+ let mut wid: u32 = 0;+ let err = _AXUIElementGetWindow(element, &mut wid);+ if err == kAXErrorSuccess && wid != 0 { Some(wid) } else { None }+}++/// Read the `AXWindows` attribute of an application element.+/// Unlike `AXChildren`, this returns the window list regardless of whether+/// the app is frontmost. Returns a Vec of retained AXUIElementRefs.+pub unsafe fn copy_ax_windows(element: AXUIElementRef) -> Vec<AXUIElementRef> {+ let attr = CFStr::new("AXWindows");+ let mut value: CFTypeRef = std::ptr::null();+ let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value);+ if err != kAXErrorSuccess || value.is_null() {+ return vec![];+ }+ let cf_array_type_id = CFArray::<CFTypeRef>::type_id();+ if core_foundation::base::CFGetTypeID(value) != cf_array_type_id {+ CFRelease(value);+ return vec![];+ }+ let arr = CFArray::<CFTypeRef>::wrap_under_create_rule(value as _);+ let ax_type_id = AXUIElementGetTypeID();+ (0..arr.len())+ .filter_map(|i| {+ let item = *arr.get(i)?;+ if core_foundation::base::CFGetTypeID(item) == ax_type_id {+ CFRetain(item);+ Some(item as AXUIElementRef)+ } else {+ None+ }+ })+ .collect()+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/ax/cache.rs b/libs/cua-driver-rs/crates/platform-macos/src/ax/cache.rsnew file mode 100644index 0000000000..f0a1c1180f--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/ax/cache.rs@@ -0,0 +1,82 @@+//! Per-(pid, window_id) element cache.+//!+//! After `get_window_state`, each actionable element's AXUIElementRef pointer+//! is cached by element_index. Subsequent `click`, `type_text`, etc. look up+//! the element_index to get the raw pointer and perform AX actions on it.+//!+//! Cache is scoped per (pid, window_id) — a new `get_window_state` call+//! for the same (pid, window_id) replaces the entire entry.+//!+//! Memory contract:+//! tree::walk_element retains each actionable element before storing its ptr.+//! CachedSnapshot::drop releases those retains so we have no AX leaks.++use super::bindings::AXUIElementRef;+use super::tree::AXNode;+use core_foundation::base::{CFRelease, CFTypeRef};+use std::collections::HashMap;+use std::sync::Mutex;++/// Key for the element cache.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct CacheKey {+ pub pid: i32,+ pub window_id: u32,+}++/// Cached snapshot for one (pid, window_id) pair.+pub struct CachedSnapshot {+ /// element_index → raw AXUIElementRef pointer (retained, as usize for Send).+ pub elements: Vec<usize>,+}++impl Drop for CachedSnapshot {+ fn drop(&mut self) {+ // Release the extra CFRetain that walk_element added for each cached ptr.+ for ptr in &self.elements {+ if *ptr != 0 {+ unsafe { CFRelease(*ptr as AXUIElementRef as CFTypeRef) };+ }+ }+ }+}++/// Global element cache.+pub struct ElementCache {+ inner: Mutex<HashMap<CacheKey, CachedSnapshot>>,+}++impl ElementCache {+ pub fn new() -> Self {+ Self { inner: Mutex::new(HashMap::new()) }+ }++ /// Replace the snapshot for (pid, window_id) with the nodes from a fresh walk.+ pub fn update(&self, pid: i32, window_id: u32, nodes: &[AXNode]) {+ let elements: Vec<usize> = nodes+ .iter()+ .filter(|n| n.element_index.is_some())+ .map(|n| n.element_ptr)+ .collect();+ let mut inner = self.inner.lock().unwrap();+ inner.insert(CacheKey { pid, window_id }, CachedSnapshot { elements });+ }++ /// Look up the raw AXUIElementRef pointer for `element_index` in (pid, window_id).+ pub fn get_element_ptr(&self, pid: i32, window_id: u32, element_index: usize) -> Option<usize> {+ let inner = self.inner.lock().unwrap();+ inner.get(&CacheKey { pid, window_id })?.elements.get(element_index).copied()+ }++ /// Number of indexed elements for (pid, window_id), or 0 if not cached.+ pub fn element_count(&self, pid: i32, window_id: u32) -> usize {+ let inner = self.inner.lock().unwrap();+ inner.get(&CacheKey { pid, window_id }).map(|s| s.elements.len()).unwrap_or(0)+ }+}++impl Default for ElementCache {+ fn default() -> Self {+ Self::new()+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/ax/mod.rs b/libs/cua-driver-rs/crates/platform-macos/src/ax/mod.rsnew file mode 100644index 0000000000..a49289fa21--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/ax/mod.rs@@ -0,0 +1,24 @@+//! macOS Accessibility (AX) API bindings and tree walker.+//!+//! # AX element indexing+//! Every *actionable* element in the tree is assigned an `element_index` (0-based,+//! depth-first). The index is stable within a single `get_window_state` snapshot+//! and is passed to `click`, `type_text`, etc. to identify the target element+//! without requiring pixel coordinates.+//!+//! # Tree format (treeMarkdown)+//! ```text+//! - [0] AXButton "OK" [actions=[press]]+//! - AXStaticText = "OK"+//! - [1] AXTextField "Name" [value="John"]+//! ```+//! - Indexed elements: `- [N] AXRole "Title" [key=val ...]`+//! - Non-indexed (no actions, not interesting): `- AXRole = "value"`+//! - 2-space indent per depth level++pub mod bindings;+pub mod tree;+pub mod cache;++pub use tree::{walk_tree, AXNode, TreeWalkResult};+pub use cache::ElementCache;diff --git a/libs/cua-driver-rs/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver-rs/crates/platform-macos/src/ax/tree.rsnew file mode 100644index 0000000000..fdeeb33483--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/ax/tree.rs@@ -0,0 +1,335 @@+//! AX tree walker: produces the treeMarkdown string and element cache.+//!+//! Format (matching libs/cua-driver exactly):+//! `INDENT- [N] AXRole "Title" [value="..." actions=[...]]`+//! `INDENT- AXStaticText = "value"` (non-indexed)+//!+//! Rules (from cua-driver reference):+//! - An element is "actionable" (gets an index) when it has ≥1 action name.+//! - Non-actionable leaf nodes with a value are rendered as `AXRole = "value"`.+//! - AXStaticText with no title/value is omitted.+//! - Tree is walked depth-first; element_index is assigned in DFS order.++use super::bindings::*;+use core_foundation::base::{CFRelease, CFRetain, CFTypeRef};++/// Maximum depth for AX tree walks. Deep menus and complex web views can+/// nest deeply; 25 covers realistic app chrome without exploding on+/// pathological trees (mirrors Swift reference implementation).+const MAX_DEPTH: usize = 25;++/// A single node in the AX tree.+#[derive(Debug, Clone)]+pub struct AXNode {+ /// 0-based index (Some = actionable, None = non-actionable display-only node)+ pub element_index: Option<usize>,+ pub role: String,+ /// AXTitle — shown as `"title"` in the tree line.+ pub title: Option<String>,+ /// AXValue — shown as `= "value"` in the tree line.+ pub value: Option<String>,+ /// AXDescription — shown as `(description)` in the tree line.+ /// Kept separate from `title` so `_find_calc_button("2")` can find+ /// Calculator buttons where AXTitle="" but AXDescription="2".+ pub description: Option<String>,+ pub identifier: Option<String>,+ pub help: Option<String>,+ pub actions: Vec<String>,+ /// The raw AXUIElementRef pointer value, for caching.+ pub element_ptr: usize,+}++pub struct TreeWalkResult {+ pub tree_markdown: String,+ pub nodes: Vec<AXNode>,+}++/// Walk the AX tree of `pid`, optionally filtered to a specific window.+///+/// `window_id` — when Some, only the AXWindow matching that CGWindowID is+/// walked (plus non-window children like the menu bar). When None, all+/// top-level children are walked.+///+/// Key background-app fix: at the application root we union `AXChildren`+/// and `AXWindows`. macOS only puts windows in `AXChildren` when the app+/// is frontmost; `AXWindows` returns the window list regardless of focus+/// state. Without this union, Safari / any backgrounded app returns an+/// empty tree.+///+/// # Safety+/// Calls macOS AX API. Must be called on a thread that has a CF run loop.+pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeWalkResult {+ let mut nodes: Vec<AXNode> = Vec::new();+ let mut lines: Vec<(usize, String)> = Vec::new(); // (depth, line)+ let mut index_counter = 0usize;++ unsafe {+ let app_elem = AXUIElementCreateApplication(pid);+ if app_elem.is_null() {+ return TreeWalkResult { tree_markdown: String::new(), nodes };+ }++ // Union AXChildren + AXWindows — the only way to see background windows.+ // AXChildren omits windows when the app isn't frontmost (AppKit limitation).+ // AXWindows returns the window list regardless of activation state.+ let from_children = copy_children(app_elem);+ let from_windows = copy_ax_windows(app_elem);++ let mut top_level = from_children;+ for w in from_windows {+ // Deduplicate by raw pointer identity.+ if !top_level.iter().any(|&e| e == w) {+ top_level.push(w);+ } else {+ // Already present — release the extra retain from copy_ax_windows.+ CFRelease(w as CFTypeRef);+ }+ }++ // Filter: keep non-window children (menu bar) + the target window.+ let walk_these: Vec<AXUIElementRef> = if let Some(wid) = window_id {+ top_level.iter().copied().filter(|&child| {+ let role = copy_string_attr(child, "AXRole").unwrap_or_default();+ if role != "AXWindow" {+ return true; // always keep menu bar and other non-window items+ }+ // Match AX window element → CGWindowID via private SPI.+ ax_get_window_id(child) == Some(wid)+ }).collect()+ } else {+ top_level.iter().copied().collect()+ };++ // Walk each top-level child at depth 0.+ for child in walk_these {+ walk_element(child, 0, &mut nodes, &mut lines, &mut index_counter);+ }++ // Release all top-level elements (copy_children / copy_ax_windows both retain).+ for child in top_level {+ CFRelease(child as CFTypeRef);+ }++ CFRelease(app_elem as CFTypeRef);+ }++ let raw_markdown = render_lines(&lines);+ let tree_markdown = if let Some(q) = query {+ filter_tree(&raw_markdown, q)+ } else {+ raw_markdown+ };++ TreeWalkResult { tree_markdown, nodes }+}++unsafe fn walk_element(+ element: AXUIElementRef,+ depth: usize,+ nodes: &mut Vec<AXNode>,+ lines: &mut Vec<(usize, String)>,+ counter: &mut usize,+) {+ if depth > MAX_DEPTH { return; }++ let role = copy_string_attr(element, "AXRole")+ .unwrap_or_else(|| "AXUnknown".into());++ // Skip pure layout containers that have no interesting content.+ if role == "AXScrollArea" || role == "AXGroup" {+ // Still recurse — children may be interesting.+ let children = copy_children(element);+ for child in children {+ walk_element(child, depth, nodes, lines, counter);+ CFRelease(child as CFTypeRef);+ }+ return;+ }++ // Keep AXTitle and AXDescription SEPARATE so that the tree format matches+ // the Swift reference: title → "title", description → (description).+ // This is critical for Calculator where AXTitle="" but AXDescription="2"+ // (digit buttons). Merging them would produce "2" (quoted) instead of (2)+ // (parens), breaking _find_calc_button which searches for "(2)".+ let title = copy_string_attr(element, "AXTitle");+ let value = copy_string_attr(element, "AXValue");+ // AXPlaceholderValue as fallback for empty text fields.+ let value = value.filter(|v| !v.trim().is_empty())+ .or_else(|| copy_string_attr(element, "AXPlaceholderValue"));+ let description = copy_string_attr(element, "AXDescription");+ let identifier = copy_string_attr(element, "AXIdentifier");+ let help = copy_string_attr(element, "AXHelp").filter(|h| !h.trim().is_empty());+ let actions = copy_action_names(element);++ let visible_title = title.as_deref().unwrap_or("").trim().to_owned();+ let visible_description = description.as_deref().unwrap_or("").trim().to_owned();+ let visible_value = value.as_deref().unwrap_or("").trim().to_owned();++ let has_content = !visible_title.is_empty()+ || !visible_description.is_empty()+ || !visible_value.is_empty();+ let is_actionable = !actions.is_empty();++ if !is_actionable && !has_content && role != "AXWindow" && role != "AXSheet" {+ let children = copy_children(element);+ for child in children {+ walk_element(child, depth + 1, nodes, lines, counter);+ CFRelease(child as CFTypeRef);+ }+ return;+ }++ let element_ptr = element as usize;+ let node = if is_actionable {+ let idx = *counter;+ *counter += 1;+ // Retain so the element stays alive in the cache after `copy_children`+ // releases the per-child ref at the end of the caller's loop.+ CFRetain(element as CFTypeRef);+ AXNode {+ element_index: Some(idx),+ role: role.clone(),+ title: if visible_title.is_empty() { None } else { Some(visible_title.clone()) },+ value: if visible_value.is_empty() { None } else { Some(visible_value.clone()) },+ description: if visible_description.is_empty() { None } else { Some(visible_description.clone()) },+ identifier: identifier.clone(),+ help: help.clone(),+ actions: actions.clone(),+ element_ptr,+ }+ } else {+ AXNode {+ element_index: None,+ role: role.clone(),+ title: if visible_title.is_empty() { None } else { Some(visible_title.clone()) },+ value: if visible_value.is_empty() { None } else { Some(visible_value.clone()) },+ description: if visible_description.is_empty() { None } else { Some(visible_description.clone()) },+ identifier: identifier.clone(),+ help: help.clone(),+ actions: vec![],+ element_ptr,+ }+ };++ let line = format_node_line(&node);+ lines.push((depth, line));+ nodes.push(node);++ let children = copy_children(element);+ for child in children {+ walk_element(child, depth + 1, nodes, lines, counter);+ CFRelease(child as CFTypeRef);+ }+}++fn format_node_line(node: &AXNode) -> String {+ let mut parts = String::new();++ // Common prefix (with or without index).+ if let Some(idx) = node.element_index {+ parts.push_str(&format!("- [{}] {}", idx, node.role));+ } else {+ parts.push_str(&format!("- {}", node.role));+ }++ // AXTitle → "title"+ if let Some(t) = &node.title {+ parts.push_str(&format!(" \"{}\"", t));+ }+ // AXValue → = "value"+ if let Some(v) = &node.value {+ parts.push_str(&format!(" = \"{}\"", v));+ }+ // AXDescription → (description) — critical for Calculator digit buttons+ // where AXTitle="" but AXDescription="2".+ if let Some(d) = &node.description {+ parts.push_str(&format!(" ({})", d));+ }++ // Bracketed metadata block (identifier, help, actions).+ if node.element_index.is_some() {+ let mut attrs: Vec<String> = Vec::new();+ if let Some(id) = &node.identifier {+ attrs.push(format!("id={}", id));+ }+ if let Some(h) = &node.help {+ attrs.push(format!("help=\"{}\"", h));+ }+ if !node.actions.is_empty() {+ let action_str = node.actions.iter()+ .map(|a| a.strip_prefix("AX").unwrap_or(a).to_lowercase())+ .collect::<Vec<_>>()+ .join(",");+ attrs.push(format!("actions=[{}]", action_str));+ }+ if !attrs.is_empty() {+ parts.push_str(" [");+ parts.push_str(&attrs.join(" "));+ parts.push(']');+ }+ }++ parts+}++fn render_lines(lines: &[(usize, String)]) -> String {+ let mut out = String::new();+ for (depth, line) in lines {+ for _ in 0..*depth {+ out.push_str(" ");+ }+ out.push_str(line);+ out.push('\n');+ }+ out+}++/// Filter the tree markdown to lines matching `query` plus their ancestor chain.+fn filter_tree(markdown: &str, query: &str) -> String {+ let needle = query.to_lowercase();+ let lines: Vec<&str> = markdown.lines().collect();++ let mut current_ancestor: Vec<&str> = Vec::new();+ let mut last_emitted_at: Vec<Option<&str>> = Vec::new();+ let mut output: Vec<&str> = Vec::new();++ for line in &lines {+ let depth = leading_indent_depth(line);++ while current_ancestor.len() <= depth {+ current_ancestor.push("");+ last_emitted_at.push(None);+ }+ for deeper in (depth + 1)..current_ancestor.len() {+ last_emitted_at[deeper] = None;+ }+ current_ancestor[depth] = line;++ if line.to_lowercase().contains(&needle) {+ for ancestor_depth in 0..depth {+ let ancestor = current_ancestor[ancestor_depth];+ if ancestor.is_empty() { continue; }+ if last_emitted_at[ancestor_depth] == Some(ancestor) { continue; }+ last_emitted_at[ancestor_depth] = Some(ancestor);+ output.push(ancestor);+ }+ last_emitted_at[depth] = Some(line);+ output.push(line);+ }+ }++ if output.is_empty() {+ return String::new();+ }+ let mut result = output.join("\n");+ result.push('\n');+ result+}++fn leading_indent_depth(line: &str) -> usize {+ let mut count = 0;+ for ch in line.chars() {+ if ch == ' ' { count += 1; } else { break; }+ }+ count / 2+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/browser/ax_page_reader.rs b/libs/cua-driver-rs/crates/platform-macos/src/browser/ax_page_reader.rsnew file mode 100644index 0000000000..bbc4f12e27--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/browser/ax_page_reader.rs@@ -0,0 +1,196 @@+//! AXPageReader: extract text and query elements from AX tree markdown strings.++pub struct AXPageReader;++#[derive(Debug, Clone)]+pub struct AXElement {+ pub role: String,+ pub title: String,+ pub value: String,+ pub description: String,+ pub index: Option<usize>,+}++/// Roles to skip when extracting plain text.+const SKIP_ROLES: &[&str] = &[+ "AXWindow", "AXApplication", "AXGroup", "AXScrollArea",+ "AXSplitGroup", "AXSplitter", "AXMenuBar", "AXMenu",+ "AXMenuBarItem", "AXUnknown",+];++/// Roles whose text content is always extracted (without filtering).+const TEXT_ROLES: &[&str] = &["AXStaticText", "AXHeading", "AXWebArea"];++impl AXPageReader {+ /// Extract readable text from an AX tree markdown string.+ pub fn extract_text(tree_markdown: &str) -> String {+ let mut lines_out: Vec<String> = Vec::new();+ let mut last: Option<String> = None;++ for line in tree_markdown.lines() {+ let Some(el) = parse_ax_line(line) else { continue };++ let is_text_role = TEXT_ROLES.contains(&el.role.as_str());+ let is_skipped = SKIP_ROLES.contains(&el.role.as_str());++ if is_skipped && !is_text_role { continue; }++ // Pick the best text: title > value > description.+ let text = best_text(&el);+ if text.is_empty() { continue; }++ // Deduplicate consecutive identical lines.+ if last.as_deref() == Some(&text) { continue; }+ last = Some(text.clone());+ lines_out.push(text);+ }++ lines_out.join("\n")+ }++ /// Query elements from AX tree markdown by CSS selector (maps to AX roles).+ pub fn query(selector: &str, tree_markdown: &str) -> Vec<AXElement> {+ let roles = css_selector_to_ax_roles(selector);+ let match_all = roles.is_empty();++ let mut result = Vec::new();+ for line in tree_markdown.lines() {+ let Some(el) = parse_ax_line(line) else { continue };+ if match_all || roles.iter().any(|r| *r == el.role) {+ result.push(el);+ }+ }+ result+ }+}++/// Parse a single AX tree markdown line into an AXElement.+///+/// Formats handled:+/// ` - [N] AXRole "title" = "value" (description)`+/// ` - AXRole "title" = "value" (description)`+/// ` - AXRole "title"`+/// ` - AXRole = "value"`+fn parse_ax_line(line: &str) -> Option<AXElement> {+ // Strip leading whitespace and "- ".+ let trimmed = line.trim();+ let rest = trimmed.strip_prefix("- ")?;++ // Optional index: `[N] `.+ let (index, rest) = if rest.starts_with('[') {+ if let Some(bracket_end) = rest.find(']') {+ let idx_str = &rest[1..bracket_end];+ let idx = idx_str.parse::<usize>().ok();+ (idx, rest[bracket_end + 1..].trim_start())+ } else {+ (None, rest)+ }+ } else {+ (None, rest)+ };++ // Role is the first whitespace-separated token.+ let (role, after_role) = split_first_word(rest);+ if role.is_empty() || !role.starts_with("AX") {+ return None;+ }+ let role = role.to_owned();+ let after_role = after_role.trim_start();++ // Parse: optional `"title"`, optional `= "value"`, optional `(description)`.+ let mut title = String::new();+ let mut value = String::new();+ let mut description = String::new();+ let mut pos = after_role;++ // Title: starts with `"`.+ if pos.starts_with('"') {+ let (t, remaining) = parse_quoted_string(pos);+ title = t;+ pos = remaining.trim_start();+ }++ // Value: `= "..."`.+ if let Some(after_eq) = pos.strip_prefix("= ") {+ let p = after_eq.trim_start();+ if p.starts_with('"') {+ let (v, remaining) = parse_quoted_string(p);+ value = v;+ pos = remaining.trim_start();+ } else {+ pos = after_eq;+ }+ } else if let Some(after_eq) = pos.strip_prefix('=') {+ let p = after_eq.trim_start();+ if p.starts_with('"') {+ let (v, remaining) = parse_quoted_string(p);+ value = v;+ pos = remaining.trim_start();+ } else {+ pos = after_eq;+ }+ }++ // Description: `(...)`.+ if pos.starts_with('(') {+ if let Some(close) = pos.find(')') {+ description = pos[1..close].to_owned();+ }+ }++ Some(AXElement { role, title, value, description, index })+}++/// Parse a double-quoted string starting at the beginning of `s`.+/// Returns (content, remainder_after_closing_quote).+fn parse_quoted_string(s: &str) -> (String, &str) {+ if !s.starts_with('"') {+ return (String::new(), s);+ }+ let s = &s[1..]; // skip opening quote+ let mut result = String::new();+ let mut chars = s.char_indices();+ loop {+ match chars.next() {+ None => return (result, ""),+ Some((i, '\\')) => {+ match chars.next() {+ Some((_, c)) => result.push(c),+ None => return (result, &s[i + 1..]),+ }+ }+ Some((i, '"')) => {+ return (result, &s[i + 1..]);+ }+ Some((_, c)) => result.push(c),+ }+ }+}++fn split_first_word(s: &str) -> (&str, &str) {+ let pos = s.find(|c: char| c.is_whitespace()).unwrap_or(s.len());+ (&s[..pos], &s[pos..])+}++fn best_text(el: &AXElement) -> String {+ if !el.title.is_empty() { return el.title.clone(); }+ if !el.value.is_empty() { return el.value.clone(); }+ if !el.description.is_empty() { return el.description.clone(); }+ String::new()+}++fn css_selector_to_ax_roles(selector: &str) -> Vec<&'static str> {+ match selector.trim() {+ "a" | "link" => vec!["AXLink"],+ "button" => vec!["AXButton"],+ "input" => vec!["AXTextField", "AXCheckBox", "AXRadioButton"],+ "h1" | "h2" | "h3"+ | "h4" | "h5" | "h6" => vec!["AXHeading"],+ "p" => vec!["AXStaticText"],+ "img" => vec!["AXImage"],+ "select" => vec!["AXPopUpButton", "AXComboBox"],+ "li" => vec!["AXStaticText"],+ "*" | "" => vec![],+ _ => vec![],+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/browser/browser_js.rs b/libs/cua-driver-rs/crates/platform-macos/src/browser/browser_js.rsnew file mode 100644index 0000000000..5dc67f1521--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/browser/browser_js.rs@@ -0,0 +1,238 @@+//! BrowserJS: run JavaScript in Chrome/Brave/Edge/Safari via AppleScript/osascript.++use std::time::Duration;+use anyhow::Context;++pub struct BrowserJs;++fn app_name_for_bundle(bundle_id: &str) -> Option<&'static str> {+ match bundle_id {+ "com.google.Chrome" => Some("Google Chrome"),+ "com.brave.Browser" => Some("Brave Browser"),+ "com.microsoft.edgemac" => Some("Microsoft Edge"),+ "com.apple.Safari" => Some("Safari"),+ _ => None,+ }+}++impl BrowserJs {+ /// Returns true if this bundle ID is a supported browser.+ pub fn supports(bundle_id: &str) -> bool {+ app_name_for_bundle(bundle_id).is_some()+ }++ /// Execute JavaScript in the browser window identified by window_id.+ pub async fn execute(javascript: &str, bundle_id: &str, window_id: u32) -> anyhow::Result<String> {+ let app_name = app_name_for_bundle(bundle_id)+ .ok_or_else(|| anyhow::anyhow!("Unsupported browser bundle: {bundle_id}"))?;++ // Get window title from CGWindowID.+ let win_title = {+ let wid = window_id;+ tokio::task::spawn_blocking(move || {+ crate::windows::all_windows()+ .into_iter()+ .find(|w| w.window_id == wid)+ .map(|w| w.title)+ .unwrap_or_default()+ }).await?+ };++ let escaped_title = escape_for_applescript_string(&win_title);+ let escaped_js = escape_js_for_applescript(javascript);++ let script = if bundle_id == "com.apple.Safari" {+ format!(+ r#"tell application "Safari"+ set matchedDoc to missing value+ repeat with d in documents+ if name of d contains "{escaped_title}" then+ set matchedDoc to d+ exit repeat+ end if+ end repeat+ if matchedDoc is missing value then+ set matchedDoc to document 1+ end if+ do JavaScript {escaped_js} in matchedDoc+end tell"#+ )+ } else {+ format!(+ r#"tell application "{app_name}"+ set matchedWindow to missing value+ repeat with w in windows+ if name of w contains "{escaped_title}" then+ set matchedWindow to w+ exit repeat+ end if+ end repeat+ if matchedWindow is missing value then+ set matchedWindow to front window+ end if+ tell active tab of matchedWindow+ execute javascript {escaped_js}+ end tell+end tell"#+ )+ };++ run_osascript(&script).await+ }++ /// Patch the browser Preferences JSON to enable Allow JavaScript from Apple Events,+ /// then relaunch the browser.+ pub async fn enable_javascript_apple_events(bundle_id: &str) -> anyhow::Result<()> {+ let app_name = app_name_for_bundle(bundle_id)+ .ok_or_else(|| anyhow::anyhow!("Unsupported browser bundle: {bundle_id}"))?;++ // Quit the browser.+ let quit_script = format!("tell application \"{app_name}\" to quit");+ let _ = tokio::process::Command::new("/usr/bin/osascript")+ .arg("-e")+ .arg(&quit_script)+ .output()+ .await;++ tokio::time::sleep(Duration::from_secs(1)).await;++ // Find profile directory.+ let home = std::env::var("HOME").unwrap_or_default();+ let profiles_dir = match bundle_id {+ "com.google.Chrome" => format!("{home}/Library/Application Support/Google/Chrome"),+ "com.brave.Browser" => format!("{home}/Library/Application Support/BraveSoftware/Brave-Browser"),+ "com.microsoft.edgemac" => format!("{home}/Library/Application Support/Microsoft Edge"),+ _ => anyhow::bail!("No profiles directory for {bundle_id}"),+ };++ // Find all Preferences files.+ let prefs_files = find_preferences_files(&profiles_dir);++ for path in prefs_files {+ if let Err(e) = patch_preferences_file(&path) {+ tracing::warn!("Failed to patch {path}: {e}");+ }+ }++ // Relaunch.+ tokio::process::Command::new("open")+ .arg("-a")+ .arg(app_name)+ .spawn()?;++ Ok(())+ }+}++fn find_preferences_files(profiles_dir: &str) -> Vec<String> {+ let Ok(entries) = std::fs::read_dir(profiles_dir) else { return vec![] };+ let mut result = Vec::new();+ for entry in entries.flatten() {+ let path = entry.path();+ let prefs = path.join("Preferences");+ if prefs.exists() {+ if let Some(s) = prefs.to_str() {+ result.push(s.to_owned());+ }+ }+ }+ result+}++fn patch_preferences_file(path: &str) -> anyhow::Result<()> {+ let content = std::fs::read_to_string(path)+ .with_context(|| format!("Reading {path}"))?;+ let mut json: serde_json::Value = serde_json::from_str(&content)+ .with_context(|| format!("Parsing {path}"))?;++ // Set browser.allow_javascript_apple_events = true.+ if let Some(obj) = json.as_object_mut() {+ let browser = obj.entry("browser")+ .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));+ if let Some(b) = browser.as_object_mut() {+ b.insert("allow_javascript_apple_events".to_owned(), serde_json::Value::Bool(true));+ }+ // Also set account_values.browser.allow_javascript_apple_events.+ let account_values = obj.entry("account_values")+ .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));+ if let Some(av) = account_values.as_object_mut() {+ let av_browser = av.entry("browser")+ .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));+ if let Some(avb) = av_browser.as_object_mut() {+ avb.insert("allow_javascript_apple_events".to_owned(), serde_json::Value::Bool(true));+ }+ }+ }++ let new_content = serde_json::to_string(&json)?;+ std::fs::write(path, new_content)+ .with_context(|| format!("Writing {path}"))?;+ Ok(())+}++/// Escape a plain string value for embedding inside an AppleScript double-quoted string.+fn escape_for_applescript_string(s: &str) -> String {+ s.replace('\\', "\\\\").replace('"', "\\\"")+}++/// Escape JS for embedding in an AppleScript string literal.+/// Multi-line JS is split by newline and concatenated with `& (ASCII character 10) &`.+fn escape_js_for_applescript(js: &str) -> String {+ let lines: Vec<String> = js.lines()+ .map(|l| {+ let escaped = l.replace('\\', "\\\\").replace('"', "\\\"");+ format!("\"{escaped}\"")+ })+ .collect();+ if lines.is_empty() {+ return "\"\"".to_owned();+ }+ if lines.len() == 1 {+ return lines[0].clone();+ }+ lines.join(" & (ASCII character 10) & ")+}++/// Write script to a temp file and run osascript with a 15s timeout.+pub async fn run_osascript(script: &str) -> anyhow::Result<String> {+ let uuid = format!("{:x}", rand_u64());+ let path = format!("/tmp/{uuid}.applescript");++ tokio::fs::write(&path, script).await+ .with_context(|| format!("Writing temp applescript to {path}"))?;++ let out = tokio::time::timeout(+ Duration::from_secs(15),+ tokio::process::Command::new("/usr/bin/osascript")+ .arg(&path)+ .output()+ ).await+ .context("osascript timed out after 15s")?+ .context("osascript failed to spawn")?;++ // Clean up.+ let _ = tokio::fs::remove_file(&path).await;++ let stderr = String::from_utf8_lossy(&out.stderr);+ if stderr.contains("turned off") || stderr.contains("AppleScript is turned off") {+ anyhow::bail!(+ "JavaScript from Apple Events is disabled. Use action=enable_javascript_apple_events \+ to enable it (requires browser restart)."+ );+ }++ if !out.status.success() && out.stdout.is_empty() {+ let msg = stderr.trim().to_owned();+ anyhow::bail!("osascript error: {msg}");+ }++ Ok(String::from_utf8_lossy(&out.stdout).trim_end_matches('\n').to_owned())+}++fn rand_u64() -> u64 {+ use std::time::{SystemTime, UNIX_EPOCH};+ let t = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default();+ // Mix in thread id for uniqueness.+ let tid = std::thread::current().id();+ t.subsec_nanos() as u64 ^ t.as_secs().wrapping_mul(6364136223846793005) ^ format!("{tid:?}").len() as u64+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/browser/cdp_client.rs b/libs/cua-driver-rs/crates/platform-macos/src/browser/cdp_client.rsnew file mode 100644index 0000000000..09ab45a941--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/browser/cdp_client.rs@@ -0,0 +1,135 @@+//! CDP (Chrome DevTools Protocol) client for Electron and other chromium-based apps.+//! Uses raw TCP for the HTTP /json endpoint and tokio-tungstenite for WebSocket.++use std::time::Duration;+use tokio::io::{AsyncReadExt, AsyncWriteExt};+use futures_util::{SinkExt, StreamExt};++pub struct CdpClient;++impl CdpClient {+ /// Returns true if a CDP endpoint is listening on `port`.+ pub async fn is_available(port: u16) -> bool {+ http_get_json(port).await.is_ok()+ }++ /// Scan the given ports and return the first one that has a "page" target.+ pub async fn find_page_target(ports: &[u16]) -> Option<u16> {+ for &port in ports {+ if let Ok(json) = http_get_json(port).await {+ if let Ok(arr) = serde_json::from_str::<serde_json::Value>(&json) {+ if let Some(list) = arr.as_array() {+ let has_page = list.iter().any(|t| {+ t.get("type").and_then(|v| v.as_str()) == Some("page")+ });+ if has_page {+ return Some(port);+ }+ }+ }+ }+ }+ None+ }++ /// Evaluate JavaScript via CDP Runtime.evaluate on the first page target.+ pub async fn evaluate(javascript: &str, port: u16) -> anyhow::Result<String> {+ // Get list of targets.+ let json = http_get_json(port).await?;+ let targets: serde_json::Value = serde_json::from_str(&json)+ .map_err(|e| anyhow::anyhow!("CDP /json parse error: {e}"))?;++ // Find first page target with a websocket URL.+ let ws_url = targets.as_array()+ .and_then(|arr| {+ arr.iter().find(|t| {+ t.get("type").and_then(|v| v.as_str()) == Some("page")+ })+ })+ .and_then(|t| t.get("webSocketDebuggerUrl").and_then(|v| v.as_str()))+ .ok_or_else(|| anyhow::anyhow!("No page target with webSocketDebuggerUrl found on port {port}"))?+ .to_owned();++ // Build the CDP request.+ let request = serde_json::json!({+ "id": 1,+ "method": "Runtime.evaluate",+ "params": {+ "expression": javascript,+ "returnByValue": true,+ "awaitPromise": true+ }+ });+ let request_str = serde_json::to_string(&request)?;++ let result = tokio::time::timeout(Duration::from_secs(10), async {+ let (mut ws_stream, _) = tokio_tungstenite::connect_async(&ws_url).await+ .map_err(|e| anyhow::anyhow!("WebSocket connect failed: {e}"))?;++ ws_stream.send(tokio_tungstenite::tungstenite::Message::Text(request_str.into())).await+ .map_err(|e| anyhow::anyhow!("WebSocket send failed: {e}"))?;++ // Read frames until we get one with id=1.+ loop {+ let msg = ws_stream.next().await+ .ok_or_else(|| anyhow::anyhow!("WebSocket stream ended without response"))??;+ let text = match msg {+ tokio_tungstenite::tungstenite::Message::Text(t) => t,+ tokio_tungstenite::tungstenite::Message::Close(_) => {+ anyhow::bail!("WebSocket closed before response");+ }+ _ => continue,+ };+ let obj: serde_json::Value = serde_json::from_str(&text)+ .map_err(|e| anyhow::anyhow!("CDP response parse error: {e}"))?;+ // Skip CDP event frames (they have "method" key but no "id").+ if obj.get("method").is_some() { continue; }+ if obj.get("id").and_then(|v| v.as_u64()) == Some(1) {+ return parse_cdp_result(&obj);+ }+ }+ }).await+ .map_err(|_| anyhow::anyhow!("CDP evaluate timed out after 10s"))??;++ Ok(result)+ }+}++fn parse_cdp_result(obj: &serde_json::Value) -> anyhow::Result<String> {+ if let Some(err) = obj.get("error") {+ let msg = err.get("message").and_then(|v| v.as_str()).unwrap_or("unknown error");+ anyhow::bail!("CDP error: {msg}");+ }+ let result = &obj["result"]["result"];+ if let Some(v) = result.get("value") {+ return Ok(match v {+ serde_json::Value::String(s) => s.clone(),+ serde_json::Value::Null => "null".to_owned(),+ other => other.to_string(),+ });+ }+ if let Some(desc) = result.get("description").and_then(|v| v.as_str()) {+ return Ok(desc.to_owned());+ }+ Ok("undefined".to_owned())+}++async fn http_get_json(port: u16) -> anyhow::Result<String> {+ let mut stream = tokio::time::timeout(+ Duration::from_millis(500),+ tokio::net::TcpStream::connect(format!("127.0.0.1:{port}"))+ ).await+ .map_err(|_| anyhow::anyhow!("TCP connect timed out"))?+ .map_err(|e| anyhow::anyhow!("TCP connect error: {e}"))?;++ let req = format!("GET /json HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n");+ stream.write_all(req.as_bytes()).await?;++ let mut buf = Vec::new();+ stream.read_to_end(&mut buf).await?;++ let s = String::from_utf8_lossy(&buf);+ // Split off HTTP headers.+ let body = s.split("\r\n\r\n").nth(1).unwrap_or("").to_string();+ Ok(body)+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/browser/electron_js.rs b/libs/cua-driver-rs/crates/platform-macos/src/browser/electron_js.rsnew file mode 100644index 0000000000..0487ab164f--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/browser/electron_js.rs@@ -0,0 +1,156 @@+//! ElectronJS: detect Electron apps and evaluate JS via CDP inspector.++use std::time::Duration;+use std::collections::HashSet;++use super::cdp_client::CdpClient;++const DEFAULT_PORTS: &[u16] = &[9222, 9223, 9224, 9225, 9230];++pub struct ElectronJs;++impl ElectronJs {+ /// Returns true if the process at `pid` is an Electron app.+ pub fn is_electron(pid: i32) -> bool {+ // Look up the bundle path from list_running_apps.+ let apps = crate::apps::list_running_apps();+ let app = apps.iter().find(|a| a.pid == pid);++ if let Some(app) = app {+ // Try to infer bundle path from app name.+ let bundle_path = find_bundle_path_for_app(&app.name);+ if let Some(bp) = bundle_path {+ let electron_fw = format!("{bp}/Contents/Frameworks/Electron Framework.framework");+ if std::path::Path::new(&electron_fw).exists() {+ return true;+ }+ }+ }++ // Fallback: check executable via lsof or ps.+ // Check if the executable itself links Electron.+ if let Ok(exe) = get_exe_path(pid) {+ // Check parent dir for Electron Framework.+ let exe_path = std::path::Path::new(&exe);+ if let Some(parent) = exe_path.parent() {+ // e.g. /Applications/Slack.app/Contents/MacOS/Slack → look for Frameworks sibling+ if let Some(macos_dir) = parent.parent() {+ let electron_fw = macos_dir.join("Frameworks/Electron Framework.framework");+ if electron_fw.exists() {+ return true;+ }+ }+ }+ }++ false+ }++ /// Evaluate JavaScript in the Electron app at `pid` via CDP.+ pub async fn execute(javascript: &str, pid: i32) -> anyhow::Result<String> {+ // 1. Check default ports for an existing CDP page target.+ if let Some(port) = CdpClient::find_page_target(DEFAULT_PORTS).await {+ return CdpClient::evaluate(javascript, port).await;+ }++ // 2. Check listening ports of this process for a CDP endpoint.+ let existing_ports = listening_ports(pid).await;+ for &port in &existing_ports {+ if CdpClient::is_available(port).await {+ return CdpClient::evaluate(javascript, port).await;+ }+ }++ // 3. Snapshot current ports, then send SIGUSR1 to open the inspector.+ let before_ports: HashSet<u16> = existing_ports.iter().cloned().collect();++ unsafe {+ libc::kill(pid, libc::SIGUSR1);+ }++ // 4. Poll for new port.+ let mut new_port: Option<u16> = None;+ for _ in 0..10 {+ tokio::time::sleep(Duration::from_millis(200)).await;+ let current = listening_ports(pid).await;+ for &p in ¤t {+ if !before_ports.contains(&p) && CdpClient::is_available(p).await {+ new_port = Some(p);+ break;+ }+ }+ if new_port.is_some() { break; }+ }++ let port = new_port.ok_or_else(|| {+ anyhow::anyhow!("Could not find Electron inspector port for pid {pid}. \+ Try launching with --inspect or --remote-debugging-port.")+ })?;++ CdpClient::evaluate(javascript, port).await+ }+}++/// Get the executable path of a process using `ps`.+fn get_exe_path(pid: i32) -> anyhow::Result<String> {+ let out = std::process::Command::new("ps")+ .args(["-p", &pid.to_string(), "-o", "comm="])+ .output()?;+ let s = String::from_utf8_lossy(&out.stdout).trim().to_owned();+ if s.is_empty() {+ anyhow::bail!("No exe for pid {pid}");+ }+ Ok(s)+}++/// Find the .app bundle path for an app by name (searches common locations).+fn find_bundle_path_for_app(name: &str) -> Option<String> {+ let dirs = [+ "/Applications",+ "/Applications/Utilities",+ "/System/Applications",+ ];+ let home = std::env::var("HOME").unwrap_or_default();+ let user_apps = format!("{home}/Applications");++ let mut all_dirs: Vec<&str> = dirs.to_vec();+ let user_str: &str = &user_apps;+ all_dirs.push(user_str);++ for dir in all_dirs {+ let candidate = format!("{dir}/{name}.app");+ if std::path::Path::new(&candidate).exists() {+ return Some(candidate);+ }+ }+ None+}++/// Return TCP listening ports for a process via `lsof`.+async fn listening_ports(pid: i32) -> Vec<u16> {+ let out = tokio::process::Command::new("lsof")+ .args([+ "-p", &pid.to_string(),+ "-iTCP", "-sTCP:LISTEN",+ "-Fn", "-P",+ ])+ .output()+ .await;++ let Ok(out) = out else { return vec![] };+ let text = String::from_utf8_lossy(&out.stdout);++ let mut ports = Vec::new();+ for line in text.lines() {+ // Lines starting with 'n' contain the address e.g. n*:9222 or n127.0.0.1:9222+ let trimmed = line.trim();+ if !trimmed.starts_with('n') { continue; }+ if let Some(colon_pos) = trimmed.rfind(':') {+ let port_str = &trimmed[colon_pos + 1..];+ if let Ok(p) = port_str.parse::<u16>() {+ ports.push(p);+ }+ }+ }+ ports+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/browser/mod.rs b/libs/cua-driver-rs/crates/platform-macos/src/browser/mod.rsnew file mode 100644index 0000000000..c3a06ec923--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/browser/mod.rs@@ -0,0 +1,11 @@+pub mod ax_page_reader;+pub mod browser_js;+pub mod cdp_client;+pub mod electron_js;+pub mod wk_web_view;++pub use browser_js::BrowserJs;+pub use cdp_client::CdpClient;+pub use electron_js::ElectronJs;+pub use wk_web_view::is_wk_web_view_app;+pub use ax_page_reader::AXPageReader;diff --git a/libs/cua-driver-rs/crates/platform-macos/src/browser/wk_web_view.rs b/libs/cua-driver-rs/crates/platform-macos/src/browser/wk_web_view.rsnew file mode 100644index 0000000000..fdd2121f74--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/browser/wk_web_view.rs@@ -0,0 +1,78 @@+//! WKWebView detection: checks whether a process embeds WebKit.++/// Returns true if the process at `pid` is a WKWebView-based app (but not Electron).+pub fn is_wk_web_view_app(pid: i32) -> bool {+ let apps = crate::apps::list_running_apps();+ let app = apps.iter().find(|a| a.pid == pid);++ let bundle_path: Option<String> = app.and_then(|a| {+ find_bundle_path_for_app_name(&a.name)+ });++ if let Some(ref bp) = bundle_path {+ // If it has Electron Framework it's Electron, not WKWebView.+ let electron_fw = format!("{bp}/Contents/Frameworks/Electron Framework.framework");+ if std::path::Path::new(&electron_fw).exists() {+ return false;+ }++ // Check for Tauri in bundle path (lowercased).+ if bp.to_lowercase().contains("tauri") {+ return true;+ }++ // Check for WebKit.framework in bundle.+ let webkit_fw = format!("{bp}/Contents/Frameworks/WebKit.framework");+ if std::path::Path::new(&webkit_fw).exists() {+ return true;+ }+ }++ // Fallback: check otool -L on the executable for WebKit linkage.+ if let Ok(exe) = get_exe_path(pid) {+ if let Ok(out) = std::process::Command::new("otool")+ .args(["-L", &exe])+ .output()+ {+ let text = String::from_utf8_lossy(&out.stdout);+ if text.contains("WebKit.framework") || text.contains("libwebkit") {+ return true;+ }+ }+ }++ false+}++fn get_exe_path(pid: i32) -> anyhow::Result<String> {+ let out = std::process::Command::new("ps")+ .args(["-p", &pid.to_string(), "-o", "comm="])+ .output()?;+ let s = String::from_utf8_lossy(&out.stdout).trim().to_owned();+ if s.is_empty() {+ anyhow::bail!("No exe for pid {pid}");+ }+ Ok(s)+}++fn find_bundle_path_for_app_name(name: &str) -> Option<String> {+ let dirs = [+ "/Applications",+ "/Applications/Utilities",+ "/System/Applications",+ ];+ let home = std::env::var("HOME").unwrap_or_default();+ let user_apps = format!("{home}/Applications");++ let mut all_dirs: Vec<&str> = dirs.to_vec();+ let user_str: &str = &user_apps;+ all_dirs.push(user_str);++ for dir in all_dirs {+ let candidate = format!("{dir}/{name}.app");+ if std::path::Path::new(&candidate).exists() {+ return Some(candidate);+ }+ }+ None+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/capture.rs b/libs/cua-driver-rs/crates/platform-macos/src/capture.rsnew file mode 100644index 0000000000..66c58eecb3--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/capture.rs@@ -0,0 +1,257 @@+//! Window / display screenshot using the macOS `screencapture` CLI tool.+//!+//! `screencapture -l <windowID> -x -o <file>` captures a single window by+//! CGWindowID to a PNG without any screen-recording permission dialog.+//!+//! `screencapture -x <file>` captures the full main display.+//!+//! For production use, the ImageIO/CGWindowListCreateImageFromArray path+//! would give lower overhead (no subprocess + temp file), but the subprocess+//! approach is simpler to implement correctly and is reliable across OS versions.++use base64::{engine::general_purpose::STANDARD as BASE64, Engine};+use std::process::Command;++/// Capture a window by its `window_id` (CGWindowID).+/// Returns raw PNG bytes or an error.+pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {+ let tmp_path = format!("/tmp/cua-driver-rs-capture-{}.png", window_id);++ let status = Command::new("screencapture")+ .args([+ "-l", &window_id.to_string(),+ "-x", // no sound+ "-o", // no shadow+ &tmp_path,+ ])+ .status()?;++ if !status.success() {+ anyhow::bail!("screencapture failed for window {window_id}");+ }++ let bytes = std::fs::read(&tmp_path)?;+ let _ = std::fs::remove_file(&tmp_path);++ if bytes.is_empty() {+ anyhow::bail!("screencapture produced empty output for window {window_id}");+ }+ Ok(bytes)+}++/// Capture a window by its `window_id` (CGWindowID).+/// Returns (base64-encoded PNG, width, height) or an error.+pub fn screenshot_window(window_id: u32) -> anyhow::Result<(String, u32, u32)> {+ let bytes = screenshot_window_bytes(window_id)?;+ let (w, h) = png_dimensions(&bytes)?;+ let b64 = BASE64.encode(&bytes);+ Ok((b64, w, h))+}++/// Capture the full main display.+/// Returns raw PNG bytes or an error.+pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {+ // Use a pid-unique path so concurrent cua-driver processes don't step on each other.+ let tmp_path = format!("/tmp/cua-driver-rs-display-{}.png", std::process::id());++ let status = Command::new("screencapture")+ .args(["-x", &*tmp_path])+ .status()?;++ if !status.success() {+ anyhow::bail!("screencapture failed for main display");+ }++ let bytes = std::fs::read(&tmp_path)?;+ let _ = std::fs::remove_file(&tmp_path);++ if bytes.is_empty() {+ anyhow::bail!("screencapture produced empty output for main display");+ }+ Ok(bytes)+}++/// Capture the main display and return (base64-encoded PNG, width, height).+pub fn screenshot_display() -> anyhow::Result<(String, u32, u32)> {+ let bytes = screenshot_display_bytes()?;+ let (w, h) = png_dimensions(&bytes)?;+ let b64 = BASE64.encode(&bytes);+ Ok((b64, w, h))+}++/// Convert raw PNG bytes to JPEG at the given quality (1-95).+/// Uses the `cursor_overlay::capture_utils` JPEG encoder.+pub fn png_bytes_to_jpeg(png_bytes: &[u8], quality: u8) -> anyhow::Result<Vec<u8>> {+ use image::ImageDecoder;+ let cursor = std::io::Cursor::new(png_bytes);+ let decoder = image::codecs::png::PngDecoder::new(cursor)?;+ let (w, h) = decoder.dimensions();+ let color = decoder.color_type();+ let mut buf = vec![0u8; decoder.total_bytes() as usize];+ decoder.read_image(&mut buf)?;++ // Ensure RGBA → RGB for JPEG encoding.+ let rgb_buf: Vec<u8> = if color == image::ColorType::Rgba8 {+ buf.chunks_exact(4).flat_map(|px| [px[0], px[1], px[2]]).collect()+ } else {+ buf+ };++ let mut jpeg_bytes = Vec::new();+ let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, quality);+ enc.encode(&rgb_buf, w, h, image::ColorType::Rgb8.into())?;+ Ok(jpeg_bytes)+}++/// Downscale `png_bytes` so neither dimension exceeds `max_dim`.+/// If `max_dim == 0` or the image already fits, returns the original bytes unchanged.+pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> anyhow::Result<Vec<u8>> {+ if max_dim == 0 {+ return Ok(png_bytes.to_vec());+ }+ let (w, h) = png_dimensions(png_bytes)?;+ if w <= max_dim && h <= max_dim {+ return Ok(png_bytes.to_vec());+ }+ // Determine scale factor to fit within max_dim x max_dim.+ let scale = (max_dim as f64) / (w.max(h) as f64);+ let new_w = (w as f64 * scale).round() as u32;+ let new_h = (h as f64 * scale).round() as u32;++ use image::ImageDecoder;+ let cursor = std::io::Cursor::new(png_bytes);+ let decoder = image::codecs::png::PngDecoder::new(cursor)?;+ let color = decoder.color_type();+ let mut buf = vec![0u8; decoder.total_bytes() as usize];+ decoder.read_image(&mut buf)?;++ let img = match color {+ image::ColorType::Rgba8 => {+ image::DynamicImage::ImageRgba8(+ image::ImageBuffer::from_raw(w, h, buf)+ .ok_or_else(|| anyhow::anyhow!("invalid RGBA buffer"))?,+ )+ }+ image::ColorType::Rgb8 => {+ image::DynamicImage::ImageRgb8(+ image::ImageBuffer::from_raw(w, h, buf)+ .ok_or_else(|| anyhow::anyhow!("invalid RGB buffer"))?,+ )+ }+ _ => anyhow::bail!("unsupported color type for resize: {color:?}"),+ };++ let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3);+ let mut out = Vec::new();+ resized.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?;+ Ok(out)+}++/// Draw a red crosshair at pixel (cx, cy) on a PNG image and write to `path`.+/// Used by `click`'s `debug_image_out` param to verify coordinate spaces.+/// The crosshair uses top-left-origin coords matching the click tool's convention.+pub fn write_crosshair_png(+ png_bytes: &[u8],+ cx: f64,+ cy: f64,+ path: &str,+) -> anyhow::Result<()> {+ use image::{ImageDecoder, DynamicImage};++ let cursor = std::io::Cursor::new(png_bytes);+ let decoder = image::codecs::png::PngDecoder::new(cursor)?;+ let (w, h) = decoder.dimensions();+ let color = decoder.color_type();+ let mut buf = vec![0u8; decoder.total_bytes() as usize];+ decoder.read_image(&mut buf)?;++ let mut img: DynamicImage = match color {+ image::ColorType::Rgba8 => DynamicImage::ImageRgba8(+ image::ImageBuffer::from_raw(w, h, buf)+ .ok_or_else(|| anyhow::anyhow!("invalid RGBA buffer"))?,+ ),+ image::ColorType::Rgb8 => DynamicImage::ImageRgb8(+ image::ImageBuffer::from_raw(w, h, buf)+ .ok_or_else(|| anyhow::anyhow!("invalid RGB buffer"))?,+ ).into(),+ _ => anyhow::bail!("unsupported color type for crosshair: {color:?}"),+ };+ // Ensure RGBA for drawing.+ let mut img = img.to_rgba8();++ // Crosshair geometry.+ let ring_r = (w as f64 / 80.0).max(6.0) as i32;+ let arm_len = (w as f64 / 40.0).max(12.0) as i32;+ let line_w = ((w as f64 / 400.0).max(1.5)) as i32;+ let red = image::Rgba([255u8, 26, 26, 242]);+ let cx = cx as i32;+ let cy = cy as i32;++ // Draw horizontal + vertical arms.+ for lw in 0..=line_w {+ let off = lw - line_w / 2;+ for dx in -arm_len..=arm_len {+ if let Some(p) = img.get_pixel_mut_checked(+ (cx + dx).clamp(0, w as i32 - 1) as u32,+ (cy + off).clamp(0, h as i32 - 1) as u32,+ ) { *p = red; }+ }+ for dy in -arm_len..=arm_len {+ if let Some(p) = img.get_pixel_mut_checked(+ (cx + off).clamp(0, w as i32 - 1) as u32,+ (cy + dy).clamp(0, h as i32 - 1) as u32,+ ) { *p = red; }+ }+ }++ // Draw ring (stroke circle).+ let steps = (ring_r * 12).max(48) as usize;+ for i in 0..steps {+ let theta = 2.0 * std::f64::consts::PI * i as f64 / steps as f64;+ let rx = (cx as f64 + ring_r as f64 * theta.cos()) as i32;+ let ry = (cy as f64 + ring_r as f64 * theta.sin()) as i32;+ for lw in 0..=line_w {+ let off = lw - line_w / 2;+ for dx in 0..=1 {+ let fx = (rx + off + dx).clamp(0, w as i32 - 1) as u32;+ let fy = (ry + off).clamp(0, h as i32 - 1) as u32;+ *img.get_pixel_mut(fx, fy) = red;+ }+ }+ }++ // Write PNG to path.+ let path = if let Some(rest) = path.strip_prefix('~') {+ format!("{}{}", std::env::var("HOME").unwrap_or_default(), rest)+ } else {+ path.to_owned()+ };+ img.save_with_format(&path, image::ImageFormat::Png)?;+ Ok(())+}++/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return the modified PNG bytes.+/// Used by recording's click-marker callback to produce click.png.+pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> anyhow::Result<Vec<u8>> {+ use std::io::Cursor;+ let tmp_path = format!("/tmp/cua-driver-rs-clickmarker-{}.png", std::process::id());+ write_crosshair_png(png_bytes, cx, cy, &tmp_path)?;+ let out = std::fs::read(&tmp_path)?;+ let _ = std::fs::remove_file(&tmp_path);+ Ok(out)+}++/// Parse width and height from a PNG file's IHDR chunk.+pub fn png_dimensions(data: &[u8]) -> anyhow::Result<(u32, u32)> {+ // PNG signature: 8 bytes, IHDR chunk: 4-byte length, "IHDR", 4-byte width, 4-byte height+ if data.len() < 24 {+ anyhow::bail!("PNG data too small");+ }+ // Signature check+ if &data[0..8] != b"\x89PNG\r\n\x1a\n" {+ anyhow::bail!("Not a PNG file");+ }+ let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]);+ let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]);+ Ok((w, h))+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/cursor/mod.rs b/libs/cua-driver-rs/crates/platform-macos/src/cursor/mod.rsnew file mode 100644index 0000000000..25b4c3ba4c--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/cursor/mod.rs@@ -0,0 +1,14 @@+//! Agent cursor overlay — transparent click-through window.+//!+//! Public surface:+//! - `overlay::init(cfg)` — called from `register_tools_with_cursor`+//! - `overlay::run_on_main_thread()` — called from `main()` on the main thread+//! - `overlay::send_command(cmd)` — called from tool implementations++pub mod state;+pub mod overlay;++// Re-export the legacy per-instance cursor state (used by tools for multi-cursor tracking).+pub use state::{CursorRegistry, CursorState};+// Note: `state::CursorConfig` (the old runtime config) is intentionally not re-exported+// at this level to avoid conflicting with `cursor_overlay::CursorConfig` (the CLI/shared config).diff --git a/libs/cua-driver-rs/crates/platform-macos/src/cursor/overlay.rs b/libs/cua-driver-rs/crates/platform-macos/src/cursor/overlay.rsnew file mode 100644index 0000000000..1091d8a8ba--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/cursor/overlay.rs@@ -0,0 +1,968 @@+//! macOS agent-cursor overlay — transparent click-through NSWindow.+//!+//! ## Architecture+//!+//! The MCP/tokio server runs on a **background thread** (spawned in+//! `cua-driver/src/main.rs`). AppKit MUST run on the **main thread**.+//! The two sides communicate through a global lock-free channel:+//!+//! - MCP tool calls → `send_command(OverlayCommand)` → `CMD_TX` (SyncSender)+//! - main thread → `run_on_main_thread()` → drains `CMD_RX` every frame+//!+//! The render loop uses a GCD background thread at ~60 fps. Each tick it+//! renders the animation state into a `tiny_skia::Pixmap`, converts to a+//! `CGImage`, and dispatches `CALayer.setContents` back to the main queue.+//!+//! ## Coordinate system+//!+//! All coordinates are **screen points** with the **top-left origin**+//! (matching `OverlayCommand::MoveTo` and AX element coordinates). The+//! NSWindow covers `NSScreen.mainScreen.frame` which AppKit places with+//! a bottom-left origin, so we flip Y when drawing into the Pixmap.++use std::ffi::c_void;+use std::sync::{Mutex, OnceLock};+use std::time::{Duration, Instant};++use cursor_overlay::{+ CursorConfig, CursorShape, MotionConfig, OverlayCommand, Palette, PathPlanner, PathState,+ PlannedPath,+};++// ── Arrival-signal channel ────────────────────────────────────────────────++static ARRIVAL_TX: Mutex<Option<tokio::sync::oneshot::Sender<()>>> = Mutex::new(None);++// ── Global overlay state ──────────────────────────────────────────────────++static CMD_TX: OnceLock<std::sync::mpsc::SyncSender<OverlayCommand>> = OnceLock::new();+// Single-consumer slot; receiver is moved into run_on_main_thread().+static CMD_RX_CELL: Mutex<Option<std::sync::mpsc::Receiver<OverlayCommand>>> = Mutex::new(None);+static RENDER: Mutex<Option<RenderState>> = Mutex::new(None);++/// Initialise global overlay state (call once, before run_on_main_thread).+pub fn init(cfg: CursorConfig) {+ let (tx, rx) = std::sync::mpsc::sync_channel(4096);+ let _ = CMD_TX.set(tx);+ *CMD_RX_CELL.lock().unwrap() = Some(rx);+ *RENDER.lock().unwrap() = Some(RenderState::new(cfg));+}++/// Send a command from any thread (MCP tool, etc.). Non-blocking; drops if+/// the channel is full (old commands are less important than new ones).+pub fn send_command(cmd: OverlayCommand) {+ if let Some(tx) = CMD_TX.get() {+ let _ = tx.try_send(cmd);+ }+}++/// Return a snapshot of the current motion config (for use by set_agent_cursor_motion+/// to apply partial overrides without losing other knobs).+pub fn current_motion() -> MotionConfig {+ RENDER.lock().unwrap()+ .as_ref()+ .map(|rs| rs.motion.clone())+ .unwrap_or_default()+}++/// Animate the overlay cursor to `(x, y)` and suspend until the Dubins path+/// completes and the spring overshoot begins.+///+/// Mirrors Swift's `AgentCursor.shared.animateAndWait(to:)`.+/// Returns immediately (no animation) when:+/// - the overlay is disabled, or+/// - the cursor is still at the off-screen sentinel `(-200, -200)` — in that+/// case the caller should rely on `ClickPulse` to snap the cursor.+pub async fn animate_cursor_to(x: f64, y: f64) {+ // Check whether animation should run.+ let should_animate = {+ let guard = RENDER.lock().unwrap();+ match guard.as_ref() {+ Some(rs) if rs.cfg.enabled && rs.pos.0 > -50.0 => true,+ _ => false,+ }+ };+ if !should_animate {+ return;+ }++ // Create a one-shot channel; store the sender so the render thread can+ // fire it when the path finishes.+ let (tx, rx) = tokio::sync::oneshot::channel::<()>();+ {+ let mut guard = ARRIVAL_TX.lock().unwrap();+ // Cancel any previous waiter (superseded by new animation).+ if let Some(old_tx) = guard.take() {+ let _ = old_tx.send(());+ }+ *guard = Some(tx);+ }++ // Send the MoveTo command (click offset applied inside apply_command).+ send_command(OverlayCommand::MoveTo {+ x,+ y,+ // Arrive pointing upper-left (45°), matching the macOS system-cursor+ // convention and Swift reference (`endAngleDegrees: 45`).+ end_heading_radians: std::f64::consts::FRAC_PI_4,+ });++ // Await arrival signal (fired from render thread when Dubins path ends).+ let _ = rx.await;+}++/// Block the calling thread (must be the OS main thread) running the AppKit+/// event loop and the overlay window. Never returns normally.+///+/// Call this from `main()` after spawning the tokio background thread.+pub fn run_on_main_thread() {+ // Take the receiver.+ let rx = match CMD_RX_CELL.lock().unwrap().take() {+ Some(r) => r,+ None => {+ // init() was never called — no overlay, just spin.+ loop { std::thread::park(); }+ }+ };++ let cfg = {+ let guard = RENDER.lock().unwrap();+ match &*guard {+ Some(rs) => rs.cfg.clone(),+ None => return,+ }+ };++ if !cfg.enabled {+ loop { std::thread::park(); }+ }++ // ------------------------------------------------------------------+ // AppKit setup (all on the main thread).+ // ------------------------------------------------------------------+ unsafe { run_appkit(cfg, rx) };+}++// ── Animation / render state ──────────────────────────────────────────────++struct RenderState {+ cfg: CursorConfig,+ palette: Palette,+ motion: MotionConfig,+ /// Current rendered position (screen top-left coords).+ pos: (f64, f64),+ /// Visual heading in radians (tip direction = motion_dir + π).+ heading: f64,+ /// In-flight path (None = at rest).+ path: Option<PlannedPath>,+ dist: f64, // arc-distance travelled so far+ /// Spring settle after path arrival.+ spring: Option<Spring>,+ spring_tgt: Option<(f64, f64, f64)>, // (x, y, heading)+ /// Flash phase for ClickPulse (0..1 or None).+ click_t: Option<f64>,+ /// Window (frame) size.+ win_w: f64,+ win_h: f64,+ /// Custom cursor shape (if any); None = gradient arrow.+ shape: Option<CursorShape>,+ /// Runtime-overridden gradient colours (from set_agent_cursor_style).+ /// Each entry is [R, G, B, A]. Empty = use palette defaults.+ gradient_colors: Vec<[u8; 4]>,+ /// Runtime-overridden bloom colour (from set_agent_cursor_style).+ /// None = use palette default.+ bloom_override: Option<[u8; 4]>,+ /// Whether the overlay is visible (user-controlled).+ visible: bool,+ /// Idle-hide: elapsed seconds since last activity.+ idle_secs: f64,+ /// Idle-hide fade: 1.0 = fully visible, 0.0 = fully hidden.+ idle_alpha: f64,+ /// Currently pinned window id for z-ordering.+ pinned_wid: Option<u64>,+ /// Focus-highlight rectangle `[x, y, w, h]` in screen coords; None = not shown.+ focus_rect: Option<[f64; 4]>,+ /// Fade progress for the focus rect: 0.0 = fully visible, 1.0 = gone.+ focus_rect_t: f64,+}++#[derive(Clone, Copy)]+struct Spring {+ ox: f64, oy: f64,+ vx: f64, vy: f64,+}++impl RenderState {+ fn new(cfg: CursorConfig) -> Self {+ let palette = cfg.palette();+ let motion = cfg.motion.clone();+ let shape = cfg.shape.clone();+ RenderState {+ cfg, palette, motion, shape,+ gradient_colors: vec![],+ bloom_override: None,+ pos: (-200.0, -200.0),+ heading: std::f64::consts::FRAC_PI_4,+ path: None,+ dist: 0.0,+ spring: None,+ spring_tgt: None,+ click_t: None,+ win_w: 0.0,+ win_h: 0.0,+ visible: true,+ idle_secs: 0.0,+ idle_alpha: 1.0,+ pinned_wid: None,+ focus_rect: None,+ focus_rect_t: 1.0,+ }+ }++ fn tick(&mut self, dt: f64) -> bool {+ // Returns true if an arrival signal should be fired (path just ended).+ // Swift constants.+ const PEAK_SPEED: f64 = 900.0;+ const MIN_START_SPEED: f64 = 300.0;+ const MIN_END_SPEED: f64 = 200.0;+ const SPRING_K: f64 = 400.0;+ const SPRING_C: f64 = 17.0;+ const SPRING_OVERSHOOT: f64 = 0.8;++ let mut fire_arrival = false;++ if let Some(ref p) = self.path {+ let path_len = p.length.max(1.0);+ let u = (self.dist / path_len).min(1.0);++ // Smootherstep speed profile (normalised: peak = 1.0).+ let profile = (30.0 * u * u * (1.0 - u) * (1.0 - u)) / 1.875;+ let floor_speed = if u < 0.5 { MIN_START_SPEED } else { MIN_END_SPEED };+ let current_speed = floor_speed + (PEAK_SPEED - floor_speed) * profile;+ self.dist += current_speed * dt;++ if self.dist >= path_len {+ // Transition to spring settle.+ let end = p.sample(path_len);+ let end_heading = p.end_visual_heading;+ let vh = end.heading;+ self.spring = Some(Spring {+ ox: 0.0, oy: 0.0,+ vx: current_speed * SPRING_OVERSHOOT * vh.cos(),+ vy: current_speed * SPRING_OVERSHOOT * vh.sin(),+ });+ self.spring_tgt = Some((end.x, end.y, end_heading));+ self.pos = (end.x, end.y);+ self.heading = end_heading;+ self.path = None;+ self.dist = 0.0;+ fire_arrival = true;+ } else {+ let s: PathState = p.sample(self.dist);+ self.pos = (s.x, s.y);+ // Smooth heading rotation toward motion heading.+ let desired = s.heading + std::f64::consts::PI;+ let max_step = 14.0 * dt;+ self.heading = rotate_toward(self.heading, desired, max_step);+ }+ } else if let Some(mut s) = self.spring {+ if let Some((tx, ty, th)) = self.spring_tgt {+ let substeps = 4;+ let sdt = dt / substeps as f64;+ for _ in 0..substeps {+ s.vx += (-SPRING_K * s.ox - SPRING_C * s.vx) * sdt;+ s.vy += (-SPRING_K * s.oy - SPRING_C * s.vy) * sdt;+ s.ox += s.vx * sdt;+ s.oy += s.vy * sdt;+ }+ self.pos = (tx + s.ox, ty + s.oy);+ self.heading = th;+ if s.ox.hypot(s.oy) < 0.3 && s.vx.hypot(s.vy) < 2.0 {+ self.pos = (tx, ty);+ self.spring = None;+ } else {+ self.spring = Some(s);+ }+ }+ }++ // Advance click pulse.+ if let Some(t) = self.click_t {+ let next = t + dt * 4.0; // full pulse over 0.25s+ self.click_t = if next >= 1.0 { None } else { Some(next) };+ }++ // Advance focus-rect fade (fades out over ~600ms).+ if self.focus_rect.is_some() {+ self.focus_rect_t = (self.focus_rect_t + dt / 0.6).min(1.0);+ if self.focus_rect_t >= 1.0 {+ self.focus_rect = None;+ self.focus_rect_t = 1.0;+ }+ }++ // Idle-hide: accumulate idle time; fade out over 180ms once threshold reached.+ let idle_hide_ms = self.motion.idle_hide_ms;+ if idle_hide_ms > 0.0 {+ let moving = self.path.is_some()+ || self.spring.is_some()+ || self.click_t.is_some();+ if moving {+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ } else {+ self.idle_secs += dt;+ let fade_start = idle_hide_ms / 1000.0;+ let fade_end = fade_start + 0.18; // 180ms fade like Windows ref+ if self.idle_secs > fade_end {+ self.idle_alpha = 0.0;+ } else if self.idle_secs > fade_start {+ let t = (self.idle_secs - fade_start) / 0.18;+ self.idle_alpha = 1.0 - t.clamp(0.0, 1.0);+ }+ }+ } else {+ self.idle_alpha = 1.0;+ }++ fire_arrival+ }++ fn apply_command(&mut self, cmd: OverlayCommand) {+ match cmd {+ OverlayCommand::MoveTo { x, y, end_heading_radians } => {+ // Apply click offset (16 pt along end_heading) before planning,+ // matching Swift `moveTo(point:endAngleRadians:)`:+ // tx = clickPoint.x + cos(endAngle) * clickOffset+ // ty = clickPoint.y + sin(endAngle) * clickOffset+ const CLICK_OFFSET: f64 = 16.0;+ const TURN_RADIUS: f64 = 80.0;+ let tx = x + end_heading_radians.cos() * CLICK_OFFSET;+ let ty = y + end_heading_radians.sin() * CLICK_OFFSET;++ // If the cursor is still at the initial off-screen sentinel,+ // snap it to the offset target so the path starts on-screen.+ if self.pos.0 < -50.0 {+ self.pos = (tx, ty);+ }+ let (x0, y0) = self.pos;+ let th0 = self.heading + std::f64::consts::PI;+ let th1 = end_heading_radians + std::f64::consts::PI;+ let plan = PathPlanner::plan(+ x0, y0, th0,+ tx, ty, th1,+ end_heading_radians,+ TURN_RADIUS,+ );+ self.path = Some(plan);+ self.dist = 0.0;+ self.spring = None;+ self.spring_tgt = None;+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ }+ OverlayCommand::ClickPulse { x, y } => {+ // Only snap position on first placement (sentinel state).+ // After that the cursor stays where the animation landed.+ if self.pos.0 < -50.0 {+ // Apply same click offset so tip lands at click point.+ const CLICK_OFFSET: f64 = 16.0;+ let angle = std::f64::consts::FRAC_PI_4;+ self.pos = (x + angle.cos() * CLICK_OFFSET, y + angle.sin() * CLICK_OFFSET);+ }+ self.click_t = Some(0.0);+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ }+ OverlayCommand::SetEnabled(v) => {+ self.visible = v;+ }+ OverlayCommand::SetMotion(m) => {+ self.motion = m;+ }+ OverlayCommand::SetPalette(p) => {+ self.palette = p;+ }+ OverlayCommand::PinAbove(wid) => {+ self.pinned_wid = Some(wid);+ }+ OverlayCommand::SetShape(shape) => {+ self.shape = shape;+ }+ OverlayCommand::SetGradient { gradient_colors, bloom_color } => {+ self.gradient_colors = gradient_colors;+ self.bloom_override = bloom_color;+ }+ OverlayCommand::ShowFocusRect(rect) => {+ self.focus_rect = rect;+ self.focus_rect_t = 0.0; // reset fade to fully visible+ }+ }+ }+}++fn rotate_toward(current: f64, desired: f64, max_step: f64) -> f64 {+ let mut diff = desired - current;+ while diff > std::f64::consts::PI { diff -= 2.0 * std::f64::consts::PI; }+ while diff < -std::f64::consts::PI { diff += 2.0 * std::f64::consts::PI; }+ current + diff.clamp(-max_step, max_step)+}++// ── tiny-skia rendering ───────────────────────────────────────────────────++fn render_frame(rs: &RenderState) -> tiny_skia::Pixmap {+ let w = rs.win_w.max(1.0) as u32;+ let h = rs.win_h.max(1.0) as u32;+ let mut pm = tiny_skia::Pixmap::new(w, h).unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap());++ if !rs.visible || rs.pos.0 < -100.0 || rs.idle_alpha < 0.004 {+ return pm;+ }++ let (px, py) = rs.pos;+ let heading = rs.heading;+ let alpha_scale = rs.idle_alpha as f32;++ // --- Bloom (radial gradient behind the arrow) ---+ let bloom_r: f32 = 22.0;+ // Use runtime bloom_override if set, otherwise fall back to palette.+ let (br, bg, bb) = if let Some([r, g, b, _]) = rs.bloom_override {+ (r, g, b)+ } else {+ let [r, g, b, _] = rs.palette.bloom_inner;+ (r, g, b)+ };+ let bloom_inner = tiny_skia::Color::from_rgba8(br, bg, bb, (115.0 * alpha_scale) as u8);+ let (or_, og, ob) = if let Some([r, g, b, _]) = rs.bloom_override {+ (r, g, b)+ } else {+ let [r, g, b, _] = rs.palette.bloom_outer;+ (r, g, b)+ };+ let bloom_outer = tiny_skia::Color::from_rgba8(or_, og, ob, (26.0 * alpha_scale) as u8);+ let bloom_zero = tiny_skia::Color::from_rgba8(or_, og, ob, 0);++ let bloom_paint = {+ let mut p = tiny_skia::Paint::default();+ p.shader = tiny_skia::RadialGradient::new(+ tiny_skia::Point::from_xy(px as f32, py as f32),+ tiny_skia::Point::from_xy(px as f32, py as f32), // focal = center+ bloom_r,+ vec![+ tiny_skia::GradientStop::new(0.0, bloom_inner),+ tiny_skia::GradientStop::new(0.5, bloom_outer),+ tiny_skia::GradientStop::new(1.0, bloom_zero),+ ],+ tiny_skia::SpreadMode::Pad,+ tiny_skia::Transform::identity(),+ ).unwrap_or(tiny_skia::Shader::SolidColor(bloom_inner));+ p.anti_alias = true;+ p+ };++ let bloom_rect = tiny_skia::Rect::from_xywh(+ (px - bloom_r as f64) as f32, (py - bloom_r as f64) as f32,+ bloom_r * 2.0, bloom_r * 2.0,+ );+ if let Some(r) = bloom_rect {+ pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None);+ }++ // --- Focus rect highlight ---+ // Cyan glow border + faint fill, matching Swift AgentCursor.showFocusRect.+ if let Some([fx, fy, fw, fh]) = rs.focus_rect {+ let t = rs.focus_rect_t as f32;+ let fade = (1.0 - t) * (1.0 - t); // quadratic ease-out+ let border_a = (230.0 * fade * alpha_scale) as u8;+ let fill_a = (20.0 * fade * alpha_scale) as u8;+ // Cyan: #5EC0E8+ let (cr, cg, cb) = (0x5Eu8, 0xC0u8, 0xE8u8);++ let rect = tiny_skia::Rect::from_xywh(fx as f32, fy as f32, fw as f32, fh as f32);+ if let Some(rect) = rect {+ // Faint fill+ let mut fill_paint = tiny_skia::Paint::default();+ fill_paint.shader = tiny_skia::Shader::SolidColor(+ tiny_skia::Color::from_rgba8(cr, cg, cb, fill_a)+ );+ pm.fill_rect(rect, &fill_paint, tiny_skia::Transform::identity(), None);++ // Border stroke (2px glow)+ let mut border_paint = tiny_skia::Paint::default();+ border_paint.shader = tiny_skia::Shader::SolidColor(+ tiny_skia::Color::from_rgba8(cr, cg, cb, border_a)+ );+ border_paint.anti_alias = true;+ let stroke = tiny_skia::Stroke { width: 2.5, ..Default::default() };+ let mut pb = tiny_skia::PathBuilder::new();+ pb.push_rect(rect);+ if let Some(path) = pb.finish() {+ pm.stroke_path(&path, &border_paint, &stroke,+ tiny_skia::Transform::identity(), None);+ }+ }+ }++ // --- Click pulse ring ---+ if let Some(t) = rs.click_t {+ let ring_r = (bloom_r + 20.0 * t as f32) * (1.0 - t as f32 * 0.5);+ let alpha = ((1.0 - t) * 180.0 * alpha_scale as f64) as u8;+ let [cr, cg, cb, _] = rs.palette.cursor_mid;+ let ring_color = tiny_skia::Color::from_rgba8(cr, cg, cb, alpha);+ let mut ring_paint = tiny_skia::Paint::default();+ ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color);+ ring_paint.anti_alias = true;+ let stroke = tiny_skia::Stroke { width: 2.0, ..Default::default() };+ let mut pb = tiny_skia::PathBuilder::new();+ pb.push_circle(px as f32, py as f32, ring_r);+ if let Some(path) = pb.finish() {+ pm.stroke_path(&path, &ring_paint, &stroke, tiny_skia::Transform::identity(), None);+ }+ }++ // --- Arrow (custom shape or default gradient arrow) ---+ if let Some(ref shape) = rs.shape {+ // Custom icon: draw as a 32×32 image centered at (px, py), opacity-faded.+ let sz = 32.0_f32;+ if let Some(pix) = tiny_skia::PixmapRef::from_bytes(&shape.pixels, shape.width, shape.height) {+ let transform = tiny_skia::Transform::from_rotate_at(+ heading.to_degrees() as f32 + 180.0,+ px as f32, py as f32,+ ).pre_translate(px as f32 - sz / 2.0, py as f32 - sz / 2.0);+ let mut paint = tiny_skia::PixmapPaint::default();+ paint.opacity = alpha_scale;+ pm.draw_pixmap(0, 0, pix, &paint, transform, None);+ }+ } else {+ draw_default_arrow(+ &mut pm, &rs.palette,+ if rs.gradient_colors.is_empty() { None } else { Some(&rs.gradient_colors) },+ px as f32, py as f32, heading as f32, alpha_scale,+ );+ }++ pm+}++fn draw_default_arrow(+ pm: &mut tiny_skia::Pixmap,+ palette: &Palette,+ gradient_override: Option<&Vec<[u8; 4]>>,+ px: f32, py: f32,+ heading: f32,+ alpha_scale: f32,+) {+ // Arrow vertices (tip at +x).+ let verts: [(f32, f32); 4] = [(14.0, 0.0), (-8.0, -9.0), (-3.0, 0.0), (-8.0, 9.0)];++ // Rotate by (heading + π) so tip points in the motion direction.+ let angle = heading + std::f64::consts::PI as f32;+ let (sa, ca) = (angle.sin(), angle.cos());+ let transform_pt = |(vx, vy): (f32, f32)| -> (f32, f32) {+ (px + ca * vx - sa * vy, py + sa * vx + ca * vy)+ };++ let pts: Vec<(f32, f32)> = verts.iter().map(|&v| transform_pt(v)).collect();++ let mut pb = tiny_skia::PathBuilder::new();+ pb.move_to(pts[0].0, pts[0].1);+ for p in &pts[1..] { pb.line_to(p.0, p.1); }+ pb.close();+ let arrow_path = match pb.finish() { Some(p) => p, None => return };++ // Gradient fill: start color at tip, end color at tail.+ // Use runtime overrides when available, otherwise fall back to palette.+ let tip = pts[0];+ let tail = ((pts[1].0 + pts[3].0) / 2.0, (pts[1].1 + pts[3].1) / 2.0);+ let (r0, g0, b0) = if let Some(g) = gradient_override.and_then(|g| g.first()) {+ (g[0], g[1], g[2])+ } else {+ let [r, g, b, _] = palette.cursor_start; (r, g, b)+ };+ let (r1, g1, b1) = if let Some(g) = gradient_override.and_then(|g| g.get(1).or_else(|| g.first())) {+ (g[0], g[1], g[2])+ } else {+ let [r, g, b, _] = palette.cursor_mid; (r, g, b)+ };+ let (r2, g2, b2) = if let Some(g) = gradient_override.and_then(|g| g.last()) {+ (g[0], g[1], g[2])+ } else {+ let [r, g, b, _] = palette.cursor_end; (r, g, b)+ };++ let a = (255.0 * alpha_scale) as u8;+ let fill_paint = {+ let mut p = tiny_skia::Paint::default();+ p.shader = tiny_skia::LinearGradient::new(+ tiny_skia::Point::from_xy(tip.0, tip.1),+ tiny_skia::Point::from_xy(tail.0, tail.1),+ vec![+ tiny_skia::GradientStop::new(0.00, tiny_skia::Color::from_rgba8(r0, g0, b0, a)),+ tiny_skia::GradientStop::new(0.53, tiny_skia::Color::from_rgba8(r1, g1, b1, a)),+ tiny_skia::GradientStop::new(1.00, tiny_skia::Color::from_rgba8(r2, g2, b2, a)),+ ],+ tiny_skia::SpreadMode::Pad,+ tiny_skia::Transform::identity(),+ ).unwrap_or(tiny_skia::Shader::SolidColor(+ tiny_skia::Color::from_rgba8(r1, g1, b1, a)+ ));+ p.anti_alias = true;+ p+ };++ pm.fill_path(&arrow_path, &fill_paint, tiny_skia::FillRule::Winding,+ tiny_skia::Transform::identity(), None);++ // White outline (faded with alpha_scale).+ let mut stroke_paint = tiny_skia::Paint::default();+ stroke_paint.shader = tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(255, 255, 255, a));+ stroke_paint.anti_alias = true;+ let stroke = tiny_skia::Stroke { width: 1.5, ..Default::default() };+ pm.stroke_path(&arrow_path, &stroke_paint, &stroke, tiny_skia::Transform::identity(), None);+}++// ── AppKit / CGImage plumbing ─────────────────────────────────────────────++unsafe fn run_appkit(_cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayCommand>) {+ use objc2::runtime::AnyObject;+ use objc2::{class, msg_send};+ use objc2_foundation::{NSPoint, NSRect, NSSize};++ // ---- NSApplication ----+ // Verify main thread (MainThreadMarker is a zero-size compile-time token).+ let _mtm = objc2_foundation::MainThreadMarker::new()+ .expect("run_appkit must be called from the main thread");++ let app: *mut AnyObject = msg_send![class!(NSApplication), sharedApplication];+ // NSApplicationActivationPolicyAccessory = 1 (no Dock icon, no menu bar)+ // setActivationPolicy: returns BOOL (success), not void.+ let _: bool = msg_send![app, setActivationPolicy: 1i64];+ // Finish launching without presenting a UI (needed for NSApp.run())+ let _: () = msg_send![app, finishLaunching];++ // ---- Main screen frame ----+ let main_screen: *mut AnyObject = msg_send![class!(NSScreen), mainScreen];+ if main_screen.is_null() {+ // Headless environment (CI without display) — skip overlay entirely.+ // The MCP server continues on the background thread.+ return;+ }+ let screen_frame: NSRect = msg_send![main_screen, frame];+ let win_w = screen_frame.size.width;+ let win_h = screen_frame.size.height;++ // ---- NSWindow: single alloc + initWithContentRect:... ----+ let win: *mut AnyObject = {+ let allocated: *mut AnyObject = msg_send![class!(NSWindow), alloc];+ // NSWindowStyleMaskBorderless = 0+ // NSBackingStoreBuffered = 2+ let w: *mut AnyObject = msg_send![allocated,+ initWithContentRect: screen_frame+ styleMask: 0u64+ backing: 2u64+ defer: false+ ];+ w+ };+ if win.is_null() { return; }++ let _: () = msg_send![win, setOpaque: false];+ let clear: *mut AnyObject = msg_send![class!(NSColor), clearColor];+ let _: () = msg_send![win, setBackgroundColor: clear];+ let _: () = msg_send![win, setHasShadow: false];+ let _: () = msg_send![win, setIgnoresMouseEvents: true];+ // NSNormalWindowLevel = 0. The overlay lives at the normal window level so+ // it appears in CGWindowList layer=0 results (which agents inspect via+ // list_windows). Z-ordering above the target is managed dynamically via+ // orderWindow:relativeTo: (see dispatch_pin_above / render_loop repin).+ let _: () = msg_send![win, setLevel: 0i64];+ // NSWindowCollectionBehaviorCanJoinAllSpaces(1<<0) | FullScreenAuxiliary(1<<8) | Stationary(1<<4)+ let _: () = msg_send![win, setCollectionBehavior: (1u64 | (1<<8) | (1<<4))];+ let _: () = msg_send![win, setReleasedWhenClosed: false];+ let _: () = msg_send![win, setHidesOnDeactivate: false];++ // ---- Layer-backed content view ----+ let content_view: *mut AnyObject = msg_send![win, contentView];+ let _: () = msg_send![content_view, setWantsLayer: true];+ let layer: *mut AnyObject = msg_send![content_view, layer];++ // Set layer geometry+ let _: () = msg_send![layer, setContentsScale: 1.0_f64];+ // kCAGravityTopLeft — the string literal "topLeft"+ let gravity_ns: *mut AnyObject = msg_send![class!(NSString),+ stringWithUTF8String: b"topLeft\0".as_ptr()+ ];+ let _: () = msg_send![layer, setContentsGravity: gravity_ns];++ // ---- Update RenderState with screen size ----+ {+ let mut guard = RENDER.lock().unwrap();+ if let Some(rs) = guard.as_mut() {+ rs.win_w = win_w;+ rs.win_h = win_h;+ }+ }++ // ---- Show the window ----+ let _: () = msg_send![win, orderFrontRegardless];++ // ---- Render thread (60 fps) ----+ let layer_ptr = layer as usize;+ let win_ptr = win as usize;+ std::thread::spawn(move || {+ render_loop(layer_ptr, win_ptr, rx, win_w, win_h);+ });++ // ---- NSApplication run loop (blocks until process exits) ----+ let _: () = msg_send![app, run];+}++fn render_loop(+ layer_ptr: usize,+ win_ptr: usize,+ rx: std::sync::mpsc::Receiver<OverlayCommand>,+ _win_w: f64, _win_h: f64,+) {+ let target_frame_ms = Duration::from_millis(16); // ~60 fps+ let mut last_tick = Instant::now();+ // Repin bookkeeping: track last pinned wid and a frame counter for+ // the periodic defensive-repin (every ~60 frames ≈ 1 s).+ let mut last_pinned: Option<u64> = None;+ let mut repin_frames: u32 = 0;++ loop {+ let now = Instant::now();+ let dt = now.duration_since(last_tick).as_secs_f64().min(0.05);+ last_tick = now;++ // Drain incoming commands and tick animation.+ let (pinned_wid, fire_arrival) = {+ let mut guard = RENDER.lock().unwrap();+ match guard.as_mut() {+ Some(rs) => {+ while let Ok(cmd) = rx.try_recv() {+ rs.apply_command(cmd);+ }+ let arrived = rs.tick(dt);+ (rs.pinned_wid, arrived)+ }+ None => break,+ }+ };++ // Fire arrival signal so animate_cursor_to() can unblock.+ if fire_arrival {+ if let Ok(mut guard) = ARRIVAL_TX.lock() {+ if let Some(tx) = guard.take() {+ let _ = tx.send(());+ }+ }+ }++ // Repin: immediately on target change, then defensive every ~1 s.+ repin_frames += 1;+ let pin_changed = pinned_wid != last_pinned;+ last_pinned = pinned_wid;+ if let Some(wid) = pinned_wid {+ if pin_changed || repin_frames >= 60 {+ dispatch_pin_above(win_ptr, wid);+ repin_frames = 0;+ }+ } else if repin_frames >= 60 {+ repin_frames = 0;+ }++ // Render frame.+ let pixmap = {+ let guard = RENDER.lock().unwrap();+ if let Some(rs) = guard.as_ref() {+ render_frame(rs)+ } else {+ break;+ }+ };++ // Convert to CGImage and update layer on the main queue.+ dispatch_set_layer_contents(layer_ptr, pixmap);++ // Sleep remainder of frame budget.+ let elapsed = Instant::now().duration_since(last_tick);+ if let Some(remaining) = target_frame_ms.checked_sub(elapsed) {+ std::thread::sleep(remaining);+ }+ }+}++/// Convert a `tiny_skia::Pixmap` to a `CGImage` and set it as the contents+/// of the given `CALayer` via `dispatch_async(main_queue, ...)`.+fn dispatch_set_layer_contents(layer_ptr: usize, pixmap: tiny_skia::Pixmap) {+ // Build the CGImage from the pixmap bytes.+ let cg_image_ptr = match pixmap_to_cgimage(&pixmap) {+ Some(p) => p,+ None => return,+ };++ // Box the payload for the C callback.+ let payload = Box::new((layer_ptr, cg_image_ptr));++ // GCD symbols from libdispatch (part of the macOS system library stubs).+ // `dispatch_get_main_queue()` is an inline C function; the underlying+ // symbol is `_dispatch_main_q`, a *struct* (not a pointer).+ // We declare it as `u8` (opaque placeholder) and take its ADDRESS to+ // obtain the `dispatch_queue_t` (pointer to the struct).+ #[link(name = "dispatch", kind = "dylib")]+ extern "C" {+ // Opaque placeholder — we only ever take &_dispatch_main_q, never read it.+ static _dispatch_main_q: u8;+ fn dispatch_async_f(queue: *const c_void, context: *mut c_void,+ work: unsafe extern "C" fn(*mut c_void));+ }++ unsafe extern "C" fn set_contents_cb(ctx: *mut c_void) {+ let (layer_ptr, cg_image_ptr): (usize, usize) = *Box::from_raw(ctx as *mut _);+ let layer = layer_ptr as *mut objc2::runtime::AnyObject;+ // setContents: expects an `id` (type '@'), not a raw void pointer.+ // CGImageRef is toll-free bridged to NSObject, so we cast it to *mut AnyObject.+ let cg_id = cg_image_ptr as *mut objc2::runtime::AnyObject;+ let _: () = objc2::msg_send![layer, setContents: cg_id];+ // Release the CGImage ref we retained in pixmap_to_cgimage.+ CGImageRelease(cg_image_ptr as *mut c_void);+ }++ extern "C" { fn CGImageRelease(image: *mut c_void); }++ unsafe {+ // &_dispatch_main_q is the queue pointer (same as dispatch_get_main_queue()).+ let main_queue = &raw const _dispatch_main_q as *const c_void;+ dispatch_async_f(+ main_queue,+ Box::into_raw(payload) as *mut c_void,+ set_contents_cb,+ );+ }+}++/// Order the overlay NSWindow just above `target_wid` in the global window+/// server list. Called from the render thread; dispatches to the main queue+/// (AppKit must be used on the main thread).+///+/// `NSWindowAbove = 1`; `orderWindow:relativeTo:` accepts any CGWindowID as+/// the `relativeTo` argument — it works cross-application via CGS.+fn dispatch_pin_above(win_ptr: usize, target_wid: u64) {+ use std::ffi::c_void;++ #[link(name = "dispatch", kind = "dylib")]+ extern "C" {+ static _dispatch_main_q: u8;+ fn dispatch_async_f(+ queue: *const c_void,+ context: *mut c_void,+ work: unsafe extern "C" fn(*mut c_void),+ );+ }++ unsafe extern "C" fn reorder_cb(ctx: *mut c_void) {+ let (win_ptr, target_wid): (usize, u64) = *Box::from_raw(ctx as *mut (usize, u64));+ let win = win_ptr as *mut objc2::runtime::AnyObject;+ // NSWindowAbove = 1; relativeTo: takes NSInteger (i64 on 64-bit)+ let _: () = objc2::msg_send![win, orderWindow: 1i64 relativeTo: target_wid as i64];+ }++ let payload = Box::new((win_ptr, target_wid));+ unsafe {+ let main_queue = &raw const _dispatch_main_q as *const c_void;+ dispatch_async_f(+ main_queue,+ Box::into_raw(payload) as *mut c_void,+ reorder_cb,+ );+ }+}++/// Create a `CGImage` from a `tiny_skia::Pixmap` (premultiplied RGBA).+/// Returns a `+1` retained pointer that the caller must release.+fn pixmap_to_cgimage(pixmap: &tiny_skia::Pixmap) -> Option<usize> {+ let w = pixmap.width() as usize;+ let h = pixmap.height() as usize;+ if w == 0 || h == 0 { return None; }++ let data = pixmap.data();+ let bytes_per_row = w * 4;++ // tiny-skia produces premultiplied RGBA with bytes in memory order [R, G, B, A].+ // CGImage flag breakdown (Apple CGBitmapInfo / CGImageAlphaInfo enums):+ // kCGImageAlphaPremultipliedLast = 0x0001 → alpha is the LAST channel (RGBA)+ // kCGImageAlphaPremultipliedFirst = 0x0002 → alpha is the FIRST channel (ARGB) ← NOT what we want+ // kCGBitmapByteOrder32Big = 0x4000 → big-endian 32-bit pixel,+ // so memory order is the same as component order (bytes = [R, G, B, A]).+ // Combined: kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big = 0x4001+ // This correctly maps tiny-skia's [R, G, B, A] bytes to the display RGB channels.+ const BITMAP_INFO: u32 = 0x0001 | 0x4000; // kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big++ // Release callback: CGDataProvider calls this when it is done with the buffer.+ // `info` is the Box<Vec<u8>> we passed as the `info` argument below.+ unsafe extern "C" fn release_pixel_data(info: *mut c_void, _data: *const c_void, _size: usize) {+ // Re-box and drop to free the buffer.+ drop(Box::from_raw(info as *mut Vec<u8>));+ }++ unsafe {+ extern "C" {+ fn CGColorSpaceCreateDeviceRGB() -> *mut c_void;+ fn CGColorSpaceRelease(cs: *mut c_void);+ fn CGDataProviderCreateWithData(+ info: *mut c_void,+ data: *const c_void,+ size: usize,+ release_data: Option<unsafe extern "C" fn(*mut c_void, *const c_void, usize)>,+ ) -> *mut c_void;+ fn CGDataProviderRelease(provider: *mut c_void);+ fn CGImageCreate(+ width: usize, height: usize,+ bits_per_component: usize,+ bits_per_pixel: usize,+ bytes_per_row: usize,+ color_space: *mut c_void,+ bitmap_info: u32,+ provider: *mut c_void,+ decode: *const f64,+ should_interpolate: bool,+ intent: u32,+ ) -> *mut c_void;+ }++ // Copy the pixel data into a heap Vec; the data provider will own it+ // and free it via release_pixel_data when the CGImage is released.+ let copied: Vec<u8> = data.to_vec();+ let len = copied.len();+ let ptr = copied.as_ptr();+ // Leak the Vec into a raw Box so we can pass it as the `info` opaque pointer.+ let copied_box: *mut Vec<u8> = Box::into_raw(Box::new(copied));++ let cs = CGColorSpaceCreateDeviceRGB();+ let provider = CGDataProviderCreateWithData(+ copied_box as *mut c_void,+ ptr as *const c_void,+ len,+ Some(release_pixel_data), // frees copied_box when provider is released+ );+ let img = CGImageCreate(+ w, h,+ 8, // bits_per_component+ 32, // bits_per_pixel+ bytes_per_row,+ cs,+ BITMAP_INFO,+ provider,+ std::ptr::null(),+ false,+ 0, // kCGRenderingIntentDefault+ );++ CGColorSpaceRelease(cs);+ CGDataProviderRelease(provider);+ // Do NOT drop copied_box here — release_pixel_data owns it now.++ if img.is_null() { None } else { Some(img as usize) }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/cursor/state.rs b/libs/cua-driver-rs/crates/platform-macos/src/cursor/state.rsnew file mode 100644index 0000000000..9ba06c7171--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/cursor/state.rs@@ -0,0 +1,112 @@+//! Cursor configuration and runtime state.++use serde::{Deserialize, Serialize};+use std::collections::HashMap;+use std::sync::Mutex;++/// Per-instance cursor configuration.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct CursorConfig {+ /// Instance identifier for multi-cursor use cases. Default: "default".+ pub cursor_id: String,+ /// Built-in icon name ("arrow", "crosshair", "hand") or path to PNG/SVG.+ pub cursor_icon: Option<String>,+ /// Hex color (e.g. "#00FFFF") or CSS name for the glow/indicator.+ pub cursor_color: Option<String>,+ /// Short label rendered near the cursor dot.+ pub cursor_label: Option<String>,+ /// Dot radius in points. Default: 16.+ pub cursor_size: Option<f64>,+ /// Opacity 0.0–1.0. Default: 0.85.+ pub cursor_opacity: Option<f64>,+ /// Whether the overlay is currently visible.+ pub enabled: bool,+}++impl Default for CursorConfig {+ fn default() -> Self {+ Self {+ cursor_id: "default".into(),+ cursor_icon: None,+ cursor_color: Some("#00FFFF".into()),+ cursor_label: None,+ cursor_size: Some(16.0),+ cursor_opacity: Some(0.85),+ enabled: true,+ }+ }+}++/// Runtime position of the agent cursor.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct CursorPosition {+ pub x: f64,+ pub y: f64,+}++/// Full state for a cursor instance.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct CursorState {+ pub config: CursorConfig,+ pub position: Option<CursorPosition>,+}++/// Global registry of cursor instances, keyed by `cursor_id`.+pub struct CursorRegistry {+ inner: Mutex<HashMap<String, CursorState>>,+}++impl CursorRegistry {+ pub fn new() -> Self {+ let mut map = HashMap::new();+ let default = CursorState {+ config: CursorConfig::default(),+ position: None,+ };+ map.insert("default".into(), default);+ Self { inner: Mutex::new(map) }+ }++ pub fn get_or_create(&self, cursor_id: &str) -> CursorState {+ let mut inner = self.inner.lock().unwrap();+ inner.entry(cursor_id.to_owned()).or_insert_with(|| CursorState {+ config: CursorConfig { cursor_id: cursor_id.to_owned(), ..Default::default() },+ position: None,+ }).clone()+ }++ pub fn update_config(&self, config: CursorConfig) {+ let mut inner = self.inner.lock().unwrap();+ let entry = inner.entry(config.cursor_id.clone()).or_insert_with(|| CursorState {+ config: config.clone(),+ position: None,+ });+ entry.config = config;+ }++ pub fn update_position(&self, cursor_id: &str, x: f64, y: f64) {+ let mut inner = self.inner.lock().unwrap();+ if let Some(state) = inner.get_mut(cursor_id) {+ state.position = Some(CursorPosition { x, y });+ }+ }++ pub fn set_enabled(&self, cursor_id: &str, enabled: bool) {+ let mut inner = self.inner.lock().unwrap();+ let state = inner.entry(cursor_id.to_owned()).or_insert_with(|| CursorState {+ config: CursorConfig { cursor_id: cursor_id.to_owned(), ..Default::default() },+ position: None,+ });+ state.config.enabled = enabled;+ }++ pub fn all_states(&self) -> Vec<CursorState> {+ self.inner.lock().unwrap().values().cloned().collect()+ }+}++impl Default for CursorRegistry {+ fn default() -> Self {+ Self::new()+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/input/ax_actions.rs b/libs/cua-driver-rs/crates/platform-macos/src/input/ax_actions.rsnew file mode 100644index 0000000000..18f2cddea2--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/input/ax_actions.rs@@ -0,0 +1,55 @@+//! AX action dispatch — the preferred click/interaction path for indexed elements.++use crate::ax::bindings::*;++/// Perform an AX action on a cached element.+pub fn perform_ax_action(element_ptr: usize, action: &str) -> anyhow::Result<()> {+ let ax_action = map_action(action);+ let err = unsafe {+ perform_action(element_ptr as AXUIElementRef, ax_action)+ };++ if err == kAXErrorSuccess {+ Ok(())+ } else {+ anyhow::bail!("AXUIElementPerformAction({action}) failed with error {err}")+ }+}++fn map_action(action: &str) -> &'static str {+ match action.to_lowercase().as_str() {+ "press" | "click" => "AXPress",+ "show_menu" | "right_click" | "rightclick" => "AXShowMenu",+ "pick" => "AXPick",+ "confirm" => "AXConfirm",+ "cancel" => "AXCancel",+ "open" => "AXOpen",+ _ => "AXPress",+ }+}++/// Set AXFocused=true on an element (for pre-focusing before key press).+pub fn focus_element(element_ptr: usize) -> anyhow::Result<()> {+ let err = unsafe {+ set_bool_attr_true(element_ptr as AXUIElementRef, "AXFocused")+ };+ if err == kAXErrorSuccess {+ Ok(())+ } else {+ // Focus errors are often benign (element doesn't support focus).+ tracing::warn!("AXSetAttribute(AXFocused) returned {err}");+ Ok(())+ }+}++/// Set the AXValue of an element (for dropdowns, text fields, etc.).+pub fn set_ax_value(element_ptr: usize, value: &str) -> anyhow::Result<()> {+ let err = unsafe {+ set_string_attr(element_ptr as AXUIElementRef, "AXValue", value)+ };+ if err == kAXErrorSuccess {+ Ok(())+ } else {+ anyhow::bail!("AXUIElementSetAttributeValue(AXValue) failed with error {err}")+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver-rs/crates/platform-macos/src/input/keyboard.rsnew file mode 100644index 0000000000..0c465a8004--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/input/keyboard.rs@@ -0,0 +1,211 @@+//! Background keyboard synthesis via SLEventPostToPid (SkyLight SPI),+//! with fallback to the public CGEvent::post_to_pid.+//!+//! For keyboard events, `post_to_pid` attaches an `SLSEventAuthenticationMessage`+//! so Chromium-based apps accept synthetic keystrokes as trusted live input+//! (required on macOS 14+ for VS Code, Chrome, Electron apps).++use core_graphics::{+ event::{CGEvent, CGEventFlags},+ event_source::{CGEventSource, CGEventSourceStateID},+};+use foreign_types::ForeignType;++/// Press and release a single key, delivered to `pid` without stealing focus.+pub fn press_key(pid: i32, key: &str, modifiers: &[&str]) -> anyhow::Result<()> {+ // Handle "+" / "plus" → Shift+= (US keyboard layout).+ if key == "+" || key.to_lowercase() == "plus" {+ let flags = modifier_flags(&["shift"]);+ let eq_code = key_name_to_code("=")?;+ post_key(pid, eq_code, true, modifier_flags(modifiers) | flags)?;+ std::thread::sleep(std::time::Duration::from_millis(8));+ post_key(pid, eq_code, false, modifier_flags(modifiers) | flags)?;+ return Ok(());+ }++ let key_code = key_name_to_code(key)?;+ let flags = modifier_flags(modifiers);++ post_key(pid, key_code, true, flags)?;+ std::thread::sleep(std::time::Duration::from_millis(8));+ post_key(pid, key_code, false, flags)?;+ Ok(())+}++/// Type a string character-by-character to `pid`.+pub fn type_text(pid: i32, text: &str) -> anyhow::Result<()> {+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;++ for ch in text.chars() {+ let ch_str = ch.to_string();+ let down = CGEvent::new_keyboard_event(source.clone(), 0, true)+ .map_err(|_| anyhow::anyhow!("CGEvent keyboard down failed"))?;+ down.set_string(&ch_str);+ // Always zero flags: Chrome inspects the flags field to infer modifier+ // state; without this, uppercase chars (e.g. 'E') are seen as Shift+e+ // and the modifier leaks into the next character (Swift fix: event.flags = []).+ down.set_flags(CGEventFlags::CGEventFlagNull);+ post_keyboard_event(pid, &down);+ std::thread::sleep(std::time::Duration::from_millis(8));++ let up = CGEvent::new_keyboard_event(source.clone(), 0, false)+ .map_err(|_| anyhow::anyhow!("CGEvent keyboard up failed"))?;+ up.set_string(&ch_str);+ up.set_flags(CGEventFlags::CGEventFlagNull);+ post_keyboard_event(pid, &up);+ std::thread::sleep(std::time::Duration::from_millis(8));+ }+ Ok(())+}++/// Type a string character-by-character with an extra `inter_char_delay_ms`+/// pause after each character (on top of the internal 8 ms down/up gap).+pub fn type_text_with_delay(pid: i32, text: &str, inter_char_delay_ms: u64) -> anyhow::Result<()> {+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;++ for ch in text.chars() {+ let ch_str = ch.to_string();+ let down = CGEvent::new_keyboard_event(source.clone(), 0, true)+ .map_err(|_| anyhow::anyhow!("CGEvent keyboard down failed"))?;+ down.set_string(&ch_str);+ down.set_flags(CGEventFlags::CGEventFlagNull);+ post_keyboard_event(pid, &down);+ std::thread::sleep(std::time::Duration::from_millis(8));++ let up = CGEvent::new_keyboard_event(source.clone(), 0, false)+ .map_err(|_| anyhow::anyhow!("CGEvent keyboard up failed"))?;+ up.set_string(&ch_str);+ up.set_flags(CGEventFlags::CGEventFlagNull);+ post_keyboard_event(pid, &up);++ // Additional inter-character delay on top of the 8 ms internal gap.+ if inter_char_delay_ms > 0 {+ std::thread::sleep(std::time::Duration::from_millis(inter_char_delay_ms));+ } else {+ std::thread::sleep(std::time::Duration::from_millis(8));+ }+ }+ Ok(())+}++/// Send a key combination (hotkey) to `pid`.+pub fn hotkey(pid: i32, key: &str, modifiers: &[&str]) -> anyhow::Result<()> {+ press_key(pid, key, modifiers)+}++/// Send a key combination to `pid` WITHOUT the auth-message envelope.+///+/// Required for NSMenu key equivalents: with the envelope, SLEventPostToPid+/// forks onto a direct-mach path that bypasses IOHIDPostEvent — NSMenu never+/// sees those events. Without the envelope the path goes through IOHIDPostEvent+/// so NSApplication.sendEvent: dispatches NSMenu key equivalents.+pub fn hotkey_no_auth(pid: i32, key: &str, modifiers: &[&str]) -> anyhow::Result<()> {+ let key_code = key_name_to_code(key)?;+ let flags = modifier_flags(modifiers);+ post_key_no_auth(pid, key_code, true, flags)?;+ std::thread::sleep(std::time::Duration::from_millis(8));+ post_key_no_auth(pid, key_code, false, flags)?;+ Ok(())+}++/// Press and release a single key to `pid` WITHOUT the auth-message envelope.+/// Works for single keys as well as combinations (same as hotkey_no_auth for single key).+pub fn press_key_no_auth(pid: i32, key: &str, modifiers: &[&str]) -> anyhow::Result<()> {+ let key_code = key_name_to_code(key)?;+ let flags = modifier_flags(modifiers);+ post_key_no_auth(pid, key_code, true, flags)?;+ std::thread::sleep(std::time::Duration::from_millis(8));+ post_key_no_auth(pid, key_code, false, flags)?;+ Ok(())+}++/// Post a keyboard event to `pid` via SLEventPostToPid (with auth message for+/// Chromium/Electron support) or fall back to CGEvent::post_to_pid.+fn post_keyboard_event(pid: i32, event: &CGEvent) {+ let event_ptr = event.as_ptr() as *mut std::ffi::c_void;+ // attachAuthMessage = true: required for Chromium keyboard on macOS 14+.+ if !crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, true) {+ event.post_to_pid(pid as libc::pid_t);+ }+}++fn post_key(pid: i32, key_code: u16, key_down: bool, flags: CGEventFlags) -> anyhow::Result<()> {+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let event = CGEvent::new_keyboard_event(source, key_code, key_down)+ .map_err(|_| anyhow::anyhow!("CGEvent::new_keyboard_event failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ event.set_flags(flags);+ }+ post_keyboard_event(pid, &event);+ Ok(())+}++fn post_key_no_auth(pid: i32, key_code: u16, key_down: bool, flags: CGEventFlags) -> anyhow::Result<()> {+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let event = CGEvent::new_keyboard_event(source, key_code, key_down)+ .map_err(|_| anyhow::anyhow!("CGEvent::new_keyboard_event failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ event.set_flags(flags);+ }+ let event_ptr = event.as_ptr() as *mut std::ffi::c_void;+ // attach_auth_message = false → IOHIDPostEvent path → NSMenu fires+ if !crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, false) {+ event.post_to_pid(pid as libc::pid_t);+ }+ Ok(())+}++fn modifier_flags(modifiers: &[&str]) -> CGEventFlags {+ let mut flags = CGEventFlags::CGEventFlagNull;+ for m in modifiers {+ match m.to_lowercase().as_str() {+ "cmd" | "command" => flags |= CGEventFlags::CGEventFlagCommand,+ "shift" => flags |= CGEventFlags::CGEventFlagShift,+ "option" | "alt" => flags |= CGEventFlags::CGEventFlagAlternate,+ "ctrl" | "control" => flags |= CGEventFlags::CGEventFlagControl,+ "fn" => flags |= CGEventFlags::CGEventFlagSecondaryFn,+ _ => {}+ }+ }+ flags+}++fn key_name_to_code(key: &str) -> anyhow::Result<u16> {+ let code = match key.to_lowercase().as_str() {+ "return" | "enter" => 36,+ "tab" => 48,+ "space" => 49,+ "delete" | "backspace" => 51,+ "escape" | "esc" => 53,+ "command" | "cmd" => 55,+ "shift" => 56,+ "capslock" => 57,+ "option" | "alt" => 58,+ "control" | "ctrl" => 59,+ "fn" => 63,+ "home" => 115,+ "pageup" => 116,+ "del" | "forward_delete" => 117,+ "end" => 119,+ "pagedown" => 121,+ "left" | "left_arrow" => 123,+ "right" | "right_arrow" => 124,+ "down" | "down_arrow" => 125,+ "up" | "up_arrow" => 126,+ "f1" => 122, "f2" => 120, "f3" => 99, "f4" => 118, "f5" => 96,+ "f6" => 97, "f7" => 98, "f8" => 100, "f9" => 101, "f10" => 109,+ "f11" => 103, "f12" => 111,+ "a" => 0, "s" => 1, "d" => 2, "f" => 3, "h" => 4, "g" => 5, "z" => 6, "x" => 7,+ "c" => 8, "v" => 9, "b" => 11, "q" => 12, "w" => 13, "e" => 14, "r" => 15, "y" => 16,+ "t" => 17, "1" => 18, "2" => 19, "3" => 20, "4" => 21, "6" => 22, "5" => 23, "=" => 24,+ "9" => 25, "7" => 26, "-" => 27, "8" => 28, "0" => 29, "]" => 30, "o" => 31, "u" => 32,+ "[" => 33, "i" => 34, "p" => 35, "l" => 37, "j" => 38, "'" => 39, "k" => 40, ";" => 41,+ "\\" => 42, "," => 43, "/" => 44, "n" => 45, "m" => 46, "." => 47, "`" => 50,+ _ => anyhow::bail!("Unknown key name: {key}"),+ };+ Ok(code)+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-macos/src/input/mod.rsnew file mode 100644index 0000000000..3bb24ffe51--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/input/mod.rs@@ -0,0 +1,18 @@+//! Background input synthesis for macOS.+//!+//! Two strategies:+//! 1. **AX action** (`element_index` path): `AXUIElementPerformAction` — pure+//! RPC, works on hidden/backgrounded windows, no cursor move, no focus steal.+//! 2. **CGEvent / SkyLight** (`x, y` path): synthesize CGEvents and post them+//! to the target pid. Prefers `SLEventPostToPid` (SkyLight SPI) over the+//! public `CGEventPostToPid` to reach Catalyst/Chromium apps and trigger+//! the activity-monitor tickle required for live-input detection.++pub mod mouse;+pub mod keyboard;+pub mod ax_actions;+pub mod skylight;++pub use ax_actions::perform_ax_action;+pub use mouse::click_at_xy;+pub use keyboard::{press_key, type_text, hotkey};diff --git a/libs/cua-driver-rs/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver-rs/crates/platform-macos/src/input/mouse.rsnew file mode 100644index 0000000000..1e79cdbdf1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/input/mouse.rs@@ -0,0 +1,452 @@+//! Background mouse event synthesis via SLEventPostToPid (SkyLight SPI),+//! with fallback to the public CGEvent::post_to_pid for older OS releases.+//!+//! SLEventPostToPid goes through the IOHIDPostEvent path which:+//! - Triggers CGSTickleActivityMonitor (required for Catalyst / Chromium)+//! - Reaches Mac Catalyst windows that CGEventPostToPid misses+//!+//! Mouse events do NOT attach the SLSEventAuthenticationMessage envelope+//! (Swift reference: `attachAuthMessage: false`) — the auth envelope routes+//! events through a direct Mach delivery path that bypasses+//! cgAnnotatedSessionEventTap, which Chromium's window handler subscribes to.++use core_graphics::{+ event::{CGEvent, CGEventFlags, CGEventType, CGMouseButton},+ event_source::{CGEventSource, CGEventSourceStateID},+ geometry::CGPoint,+};+use foreign_types::ForeignType;++/// Left-click at `(x, y)` screen coordinates, posted to `pid`.+///+/// Window-local coordinates for backgrounded targets: if `window_local` is+/// `Some((wx, wy))`, stamps a window-local point via `CGEventSetWindowLocation`+/// SPI so WindowServer's hit-test uses the local point directly.+pub fn click_at_xy(pid: i32, x: f64, y: f64, count: usize, modifiers: &[&str]) -> anyhow::Result<()> {+ click_at_xy_inner(pid, x, y, None, None, count, modifiers)+}++/// Like `click_at_xy` but also stamps window-local `(wx, wy)` onto the event.+/// When `wid` is provided, additionally stamps Chromium routing fields+/// (f51 / f58 / f91 / f92) for better backgrounded-target delivery.+pub fn click_at_xy_with_window_local(+ pid: i32,+ x: f64, y: f64,+ wx: f64, wy: f64,+ wid: u32,+ count: usize,+ modifiers: &[&str],+) -> anyhow::Result<()> {+ click_at_xy_inner(pid, x, y, Some((wx, wy)), Some(wid), count, modifiers)+}++fn click_at_xy_inner(+ pid: i32,+ x: f64, y: f64,+ window_local: Option<(f64, f64)>,+ wid: Option<u32>,+ count: usize,+ modifiers: &[&str],+) -> anyhow::Result<()> {+ use std::time::{SystemTime, UNIX_EPOCH};++ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let point = CGPoint::new(x, y);+ let flags = parse_modifier_flags(modifiers);++ // Shared click-group ID (f58) when window_id is known: keeps all pairs+ // in one gesture so WindowServer / Chromium coalesce them correctly.+ let click_group_id: Option<i64> = wid.map(|_| {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default()+ .subsec_nanos() as i64+ });++ for pair_index in 0..count {+ let click_state = (pair_index + 1) as i64;++ let down = CGEvent::new_mouse_event(+ source.clone(),+ CGEventType::LeftMouseDown,+ point,+ CGMouseButton::Left,+ ).map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ down.set_flags(flags);+ }++ post_mouse_event(pid, &down, window_local, wid, click_group_id, click_state);+ std::thread::sleep(std::time::Duration::from_millis(16));++ let up = CGEvent::new_mouse_event(+ source.clone(),+ CGEventType::LeftMouseUp,+ point,+ CGMouseButton::Left,+ ).map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ up.set_flags(flags);+ }++ post_mouse_event(pid, &up, window_local, wid, click_group_id, click_state);++ if count > 1 {+ std::thread::sleep(std::time::Duration::from_millis(80));+ }+ }+ Ok(())+}++/// Full Chromium-compatible left-click recipe matching Swift's `clickViaAuthSignedPost`.+///+/// Sequence (Swift reference minus the `activate_without_raise` prologue — that step+/// causes window-z side effects under Rust's overlay; it is reserved for a future+/// explicit opt-in once the overlay re-pins after focus changes):+/// 1. Stamped `mouseMoved` at target coords (f0=2, cursor-state primer).+/// 2. Off-screen primer down/up at (-1, -1) (f0=1/2) — satisfies Chromium's+/// user-activation gate without hitting any DOM element.+/// 3. Target down/up pair(s) at real coordinates (f0=3/3), clickState 1→N.+///+/// All events carry:+/// - f0 = gesture phase marker (move=2, primerDown=1, primerUp=2, target=3)+/// - f1 = mouseEventClickState (1 for single, 1→2 for double)+/// - f3 = 0 (left button)+/// - f7 = 3 (NSEventSubtypeTouch)+/// - f40 = target pid (Chromium synthetic-event filter)+/// - f51 / f91 / f92 = CGWindowID (window routing)+/// - f58 = constant click-group ID across all events (gesture coalescing)+/// - `CGEventSetWindowLocation` per-event (window-local point)+///+/// Uses both SkyLight `SLEventPostToPid` AND `CGEvent::post_to_pid` (belt+suspenders)+/// for AppKit / Catalyst target coverage.+pub fn click_at_xy_chromium(+ pid: i32,+ screen_x: f64,+ screen_y: f64,+ win_local_x: f64,+ win_local_y: f64,+ wid: u32,+ count: usize,+ modifiers: &[&str],+) -> anyhow::Result<()> {+ use std::time::{SystemTime, UNIX_EPOCH};++ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let target = CGPoint::new(screen_x, screen_y);+ let off_screen = CGPoint::new(-1.0, -1.0);+ let win_local = (win_local_x, win_local_y);+ let off_local = (-1.0_f64, -1.0_f64);+ let flags = parse_modifier_flags(modifiers);+ let click_pairs = count.max(1).min(2);+ let window_id = wid as i64;++ // All 5 events share the same click-group ID so WindowServer / Chromium+ // treat the sequence as one gesture (Swift: field 58).+ let click_group_id = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default()+ .subsec_nanos() as i64;++ // Stamp required fields onto a CGEvent. All captured values are Copy so+ // this closure is Fn (callable multiple times).+ let stamp = |event: &CGEvent, local: (f64, f64), click_state: i64, phase: i64| {+ let ptr = event.as_ptr() as *mut std::ffi::c_void;+ let set = |f: u32, v: i64| { crate::input::skylight::set_integer_field(ptr, f, v); };+ set(0, phase); // kCGMouseEventNumber (gesture phase)+ set(1, click_state); // kCGMouseEventClickState+ set(3, 0); // kCGMouseEventButtonNumber (left)+ set(7, 3); // kCGMouseEventSubtype (NSEventSubtypeTouch)+ set(40, pid as i64); // Chromium synthetic-event filter+ if window_id != 0 {+ set(51, window_id); // windowNumber (NSEvent bridge equivalent)+ set(91, window_id); // kCGMouseEventWindowUnderMousePointer+ set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent+ }+ set(58, click_group_id); // click-group ID (gesture coalescing)+ crate::input::skylight::set_window_location(ptr, local.0, local.1);+ if flags != CGEventFlags::CGEventFlagNull {+ event.set_flags(flags);+ }+ };++ // Belt+suspenders: SkyLight path for Chromium/Catalyst + public API for AppKit.+ let post = |event: &CGEvent| {+ let ptr = event.as_ptr() as *mut std::ffi::c_void;+ crate::input::skylight::post_to_pid(pid as libc::pid_t, ptr, false);+ event.post_to_pid(pid as libc::pid_t);+ };++ // Step 1: mouseMoved at target (phase=2, clickState=0).+ let move_event = CGEvent::new_mouse_event(+ source.clone(), CGEventType::MouseMoved, target, CGMouseButton::Left,+ ).map_err(|_| anyhow::anyhow!("mouseMoved event creation failed"))?;+ stamp(&move_event, win_local, 0, 2);+ post(&move_event);+ std::thread::sleep(std::time::Duration::from_millis(15));++ // Step 2: off-screen primer click — opens Chromium user-activation gate+ // at an off-screen coordinate that can't hit any DOM element.+ let primer_down = CGEvent::new_mouse_event(+ source.clone(), CGEventType::LeftMouseDown, off_screen, CGMouseButton::Left,+ ).map_err(|_| anyhow::anyhow!("primer down event creation failed"))?;+ stamp(&primer_down, off_local, 1, 1);+ post(&primer_down);+ std::thread::sleep(std::time::Duration::from_millis(1));++ let primer_up = CGEvent::new_mouse_event(+ source.clone(), CGEventType::LeftMouseUp, off_screen, CGMouseButton::Left,+ ).map_err(|_| anyhow::anyhow!("primer up event creation failed"))?;+ stamp(&primer_up, off_local, 1, 2);+ post(&primer_up);+ // ≥1 frame so Chromium sees primer + target as separate gestures, not run-on.+ std::thread::sleep(std::time::Duration::from_millis(100));++ // Step 3: target click pair(s) with clickState stepped 1→N for double-click+ // coalescing (Chromium renderer coalesces pairs into dblclick when state=1→2).+ for pair_index in 1..=click_pairs {+ let click_state = pair_index as i64;++ let down = CGEvent::new_mouse_event(+ source.clone(), CGEventType::LeftMouseDown, target, CGMouseButton::Left,+ ).map_err(|_| anyhow::anyhow!("target down event creation failed"))?;+ stamp(&down, win_local, click_state, 3);+ post(&down);+ std::thread::sleep(std::time::Duration::from_millis(1));++ let up = CGEvent::new_mouse_event(+ source.clone(), CGEventType::LeftMouseUp, target, CGMouseButton::Left,+ ).map_err(|_| anyhow::anyhow!("target up event creation failed"))?;+ stamp(&up, win_local, click_state, 3);+ post(&up);++ if pair_index < click_pairs {+ // ~80 ms between pairs — under the system double-click threshold,+ // clear of coalescing back into pair N.+ std::thread::sleep(std::time::Duration::from_millis(80));+ }+ }++ Ok(())+}++/// Press-drag-release gesture from `(from_x, from_y)` to `(to_x, to_y)` in+/// screen coordinates, posted to `pid`.+///+/// `duration_ms` is the wall-clock budget for the drag path; `steps` is the+/// number of intermediate `leftMouseDragged` events linearly interpolated+/// along the path. `modifiers` are held across the entire gesture.+///+/// Like the Swift reference `MouseInput.drag`, uses the SkyLight path for+/// backgrounded-target delivery.+pub fn drag_at_xy(+ pid: i32,+ from_x: f64,+ from_y: f64,+ to_x: f64,+ to_y: f64,+ from_local: Option<(f64, f64)>,+ to_local: Option<(f64, f64)>,+ wid: Option<u32>,+ duration_ms: u64,+ steps: usize,+ modifiers: &[&str],+ button: DragButton,+) -> anyhow::Result<()> {+ use std::time::{SystemTime, UNIX_EPOCH};++ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let flags = parse_modifier_flags(modifiers);++ let (cg_button, down_type, dragged_type, up_type) = match button {+ DragButton::Left => (CGMouseButton::Left, CGEventType::LeftMouseDown,+ CGEventType::LeftMouseDragged, CGEventType::LeftMouseUp),+ DragButton::Right => (CGMouseButton::Right, CGEventType::RightMouseDown,+ CGEventType::RightMouseDragged, CGEventType::RightMouseUp),+ DragButton::Middle => (CGMouseButton::Center, CGEventType::OtherMouseDown,+ CGEventType::OtherMouseDragged, CGEventType::OtherMouseUp),+ };++ let click_group_id: Option<i64> = wid.map(|_| {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default()+ .subsec_nanos() as i64+ });++ let steps = steps.max(1);+ let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };++ // MouseDown at start.+ let from_pt = CGPoint::new(from_x, from_y);+ let down = CGEvent::new_mouse_event(source.clone(), down_type, from_pt, cg_button)+ .map_err(|_| anyhow::anyhow!("drag mouseDown failed"))?;+ if flags != CGEventFlags::CGEventFlagNull { down.set_flags(flags); }+ post_mouse_event(pid, &down, from_local, wid, click_group_id, 1);+ std::thread::sleep(std::time::Duration::from_millis(16));++ // Interpolated drag steps.+ for i in 1..=steps {+ let t = i as f64 / steps as f64;+ let ix = from_x + (to_x - from_x) * t;+ let iy = from_y + (to_y - from_y) * t;+ let il = from_local.zip(to_local).map(|((fx, fy), (tx, ty))| {+ (fx + (tx - fx) * t, fy + (ty - fy) * t)+ });+ let drag_pt = CGPoint::new(ix, iy);+ let drag = CGEvent::new_mouse_event(source.clone(), dragged_type, drag_pt, cg_button)+ .map_err(|_| anyhow::anyhow!("drag mouseDragged failed"))?;+ if flags != CGEventFlags::CGEventFlagNull { drag.set_flags(flags); }+ post_mouse_event(pid, &drag, il, wid, click_group_id, 1);+ if step_delay_ms > 0 {+ std::thread::sleep(std::time::Duration::from_millis(step_delay_ms));+ }+ }++ // MouseUp at end.+ let to_pt = CGPoint::new(to_x, to_y);+ let up = CGEvent::new_mouse_event(source.clone(), up_type, to_pt, cg_button)+ .map_err(|_| anyhow::anyhow!("drag mouseUp failed"))?;+ if flags != CGEventFlags::CGEventFlagNull { up.set_flags(flags); }+ post_mouse_event(pid, &up, to_local, wid, click_group_id, 1);++ Ok(())+}++/// Mouse button for drag gestures.+#[derive(Clone, Copy, Debug)]+pub enum DragButton {+ Left,+ Right,+ Middle,+}++/// Right-click at `(x, y)` with optional modifier keys.+pub fn right_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> {+ right_click_at_xy_inner(pid, x, y, None, modifiers)+}++/// Like `right_click_at_xy` but also stamps `CGEventSetWindowLocation` with+/// the window-local `(wx, wy)` point for better backgrounded-target delivery.+pub fn right_click_at_xy_with_window_local(+ pid: i32,+ x: f64, y: f64,+ wx: f64, wy: f64,+ modifiers: &[&str],+) -> anyhow::Result<()> {+ right_click_at_xy_inner(pid, x, y, Some((wx, wy)), modifiers)+}++fn right_click_at_xy_inner(+ pid: i32,+ x: f64, y: f64,+ window_local: Option<(f64, f64)>,+ modifiers: &[&str],+) -> anyhow::Result<()> {+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let point = CGPoint::new(x, y);+ let flags = parse_modifier_flags(modifiers);++ let down = CGEvent::new_mouse_event(+ source.clone(),+ CGEventType::RightMouseDown,+ point,+ CGMouseButton::Right,+ ).map_err(|_| anyhow::anyhow!("right mouse down failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ down.set_flags(flags);+ }+ post_mouse_event(pid, &down, window_local, None, None, 1);+ std::thread::sleep(std::time::Duration::from_millis(16));++ let up = CGEvent::new_mouse_event(+ source,+ CGEventType::RightMouseUp,+ point,+ CGMouseButton::Right,+ ).map_err(|_| anyhow::anyhow!("right mouse up failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ up.set_flags(flags);+ }+ post_mouse_event(pid, &up, window_local, None, None, 1);++ Ok(())+}++/// Post a mouse event to `pid`.+///+/// Matches Swift's `MouseInput.postBoth(_:toPid:)`:+/// - Fires `SLEventPostToPid` (SkyLight path — reaches backgrounded Chromium/Catalyst).+/// - Also fires `CGEvent::post_to_pid` (public path — lands on AppKit targets where+/// SkyLight mouse delivery drops).+/// Both are always posted in sequence regardless of whether the other succeeded.+///+/// Field stamps applied (always):+/// - f40 = `pid` (Chromium's synthetic-event filter)+/// - `CGEventSetWindowLocation` = window-local point (if provided)+///+/// Additional stamps when `wid` is provided (Chromium window-routing fields):+/// - f1 = `click_state` (kCGMouseEventClickState)+/// - f3 = 0 (kCGMouseEventButtonNumber = left)+/// - f7 = 3 (kCGMouseEventSubtype = NSEventSubtypeTouch)+/// - f51 = window_id (windowNumber, NSEvent bridge equivalent)+/// - f58 = click_group_id (gesture coalescing across pairs)+/// - f91 = window_id (kCGMouseEventWindowUnderMousePointer)+/// - f92 = window_id (kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent)+fn post_mouse_event(+ pid: i32,+ event: &CGEvent,+ window_local: Option<(f64, f64)>,+ wid: Option<u32>,+ click_group_id: Option<i64>,+ click_state: i64,+) {+ let event_ptr = event.as_ptr() as *mut std::ffi::c_void;++ // Stamp window-local point for backgrounded window targeting.+ if let Some((wx, wy)) = window_local {+ crate::input::skylight::set_window_location(event_ptr, wx, wy);+ }++ // Chromium window-routing fields — stamp when window_id is known.+ if let (Some(wid), Some(cgid)) = (wid, click_group_id) {+ let window_id = wid as i64;+ let set = |f: u32, v: i64| { crate::input::skylight::set_integer_field(event_ptr, f, v); };+ set(1, click_state); // kCGMouseEventClickState+ set(3, 0); // kCGMouseEventButtonNumber (left)+ set(7, 3); // kCGMouseEventSubtype (NSEventSubtypeTouch)+ set(51, window_id); // windowNumber+ set(58, cgid); // click-group ID (gesture coalescing)+ set(91, window_id); // kCGMouseEventWindowUnderMousePointer+ set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent+ }++ // Always stamp f40 = target pid (Chromium synthetic-event filter).+ crate::input::skylight::set_integer_field(event_ptr, 40, pid as i64);++ // SkyLight path: activity-monitor tickle → reaches Catalyst/Chromium.+ // Mouse events skip the auth-message envelope (Swift: attachAuthMessage: false).+ crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, false);++ // Public path: delivers to AppKit targets where SkyLight mouse drops.+ // Belt+suspenders — both fire unconditionally (matches Swift `postBoth`).+ event.post_to_pid(pid as libc::pid_t);+}++fn parse_modifier_flags(modifiers: &[&str]) -> CGEventFlags {+ let mut flags = CGEventFlags::CGEventFlagNull;+ for m in modifiers {+ match m.to_lowercase().as_str() {+ "cmd" | "command" => flags |= CGEventFlags::CGEventFlagCommand,+ "shift" => flags |= CGEventFlags::CGEventFlagShift,+ "option" | "alt" => flags |= CGEventFlags::CGEventFlagAlternate,+ "ctrl" | "control" => flags |= CGEventFlags::CGEventFlagControl,+ _ => {}+ }+ }+ flags+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver-rs/crates/platform-macos/src/input/skylight.rsnew file mode 100644index 0000000000..cedda2bd73--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/input/skylight.rs@@ -0,0 +1,434 @@+//! SkyLight SPI bridge — Rust port of Swift's `SkyLightEventPost`.+//!+//! Two-layer story (matches Swift reference exactly):+//!+//! 1. **Post path** — `SLEventPostToPid` goes through `SLEventPostToPSN` →+//! `CGSTickleActivityMonitor` → `SLSUpdateSystemActivityWithLocation` →+//! `IOHIDPostEvent`. The public `CGEventPostToPid` skips the activity-monitor+//! tickle so Chromium/Catalyst targets don't accept those events as live input.+//!+//! 2. **Authentication** (keyboard only) — on macOS 14+, WindowServer gates+//! synthetic keyboard events on Chromium-like targets on an attached+//! `SLSEventAuthenticationMessage`. We build one via the ObjC factory and+//! attach it with `SLEventSetAuthenticationMessage` before posting.+//!+//! All symbols are resolved once at first use via `dlopen` + `dlsym`.+//! If anything fails to resolve the functions return `false` and callers+//! fall back to the public `CGEvent::post_to_pid`.++use std::ffi::{CStr, c_void};+use std::os::raw::{c_int, c_uint, c_char};+use std::sync::OnceLock;+use libc::pid_t;++// ── Function-pointer typedefs ──────────────────────────────────────────────++/// `void SLEventPostToPid(pid_t, CGEventRef)`+type PostToPidFn = unsafe extern "C" fn(pid_t, *mut c_void);++/// `void SLEventSetAuthenticationMessage(CGEventRef, id)`+type SetAuthMsgFn = unsafe extern "C" fn(*mut c_void, *mut c_void);++/// `void CGEventSetWindowLocation(CGEventRef, double x, double y)`+///+/// NOTE: CGPoint on 64-bit ARM/x86 is two f64 values packed consecutively.+/// We pass them as two separate f64 arguments which has identical ABI.+type SetWindowLocFn = unsafe extern "C" fn(*mut c_void, f64, f64);++/// `void SLEventSetIntegerValueField(CGEventRef, uint32_t field, int64_t value)`+type SetIntFieldFn = unsafe extern "C" fn(*mut c_void, u32, i64);++/// `uint32_t CGSMainConnectionID(void)`+type ConnectionIDFn = unsafe extern "C" fn() -> u32;++// ── NSMenu shortcut activation SPIs ──────────────────────────────────────────++/// `OSStatus SLPSSetFrontProcessWithOptions(const void *psn, uint32_t windowID, uint32_t options)`+type SetFrontProcessFn = unsafe extern "C" fn(*const c_void, u32, u32) -> i32;++/// `OSStatus SLSGetWindowOwner(uint32_t cid, uint32_t wid, uint32_t *out_cid)`+type GetWindowOwnerFn = unsafe extern "C" fn(u32, u32, *mut u32) -> i32;++/// `OSStatus SLSGetConnectionPSN(uint32_t cid, void *psn)`+type GetConnectionPSNFn = unsafe extern "C" fn(u32, *mut c_void) -> i32;++// ── Focus-without-raise SPIs ──────────────────────────────────────────────────++/// `OSStatus SLPSPostEventRecordTo(const void *psn, const uint8_t *bytes)`+/// Posts a 248-byte synthetic event record into the target process's Carbon+/// event queue. Build the buffer with bytes[0x04]=0xf8, bytes[0x08]=0x0d,+/// target window id at bytes 0x3c–0x3f (little-endian), focus/defocus marker+/// at bytes[0x8a] (0x01 = focus, 0x02 = defocus), all other bytes zero.+type PostEventRecordToFn = unsafe extern "C" fn(*const c_void, *const u8) -> i32;++/// `OSStatus _SLPSGetFrontProcess(void *psn)`+/// Writes the current frontmost process's 8-byte PSN into `psn`.+type GetFrontProcessFn = unsafe extern "C" fn(*mut c_void) -> i32;++/// `OSStatus GetProcessForPID(pid_t, void *psn)`+/// Deprecated but still resolves. Writes the target pid's 8-byte PSN.+type GetProcessForPIDFn = unsafe extern "C" fn(pid_t, *mut c_void) -> i32;++/// Factory: `+[SLSEventAuthenticationMessage messageWithEventRecord:pid:version:]`+/// ObjC send: `(id self, SEL _cmd, void* record, int32 pid, uint32 version) -> id`+type FactoryMsgSendFn = unsafe extern "C" fn(+ *mut c_void, // Class (receiver)+ *mut c_void, // SEL+ *mut c_void, // SLSEventRecord*+ c_int, // pid+ c_uint, // version+) -> *mut c_void;++// ── Symbol resolution ──────────────────────────────────────────────────────++/// Load SkyLight once so all dlsym lookups via RTLD_DEFAULT find it.+fn ensure_skylight_loaded() {+ static LOADED: OnceLock<()> = OnceLock::new();+ LOADED.get_or_init(|| {+ let path = b"/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight\0";+ unsafe {+ libc::dlopen(path.as_ptr() as *const c_char, libc::RTLD_LAZY | libc::RTLD_GLOBAL);+ }+ });+}++/// Look up a symbol by name via RTLD_DEFAULT (after loading SkyLight).+/// Returns `None` when the symbol doesn't resolve.+fn find_sym(name: &[u8]) -> Option<*mut c_void> {+ ensure_skylight_loaded();+ let ptr = unsafe {+ libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr() as *const c_char)+ };+ if ptr.is_null() { None } else { Some(ptr) }+}++/// Reinterpret a raw symbol pointer as a function pointer of type `T`.+/// Safety: caller guarantees T matches the symbol's actual signature.+unsafe fn as_fn<T: Copy>(ptr: *mut c_void) -> T {+ std::mem::transmute_copy::<*mut c_void, T>(&ptr)+}++// ── Lazily-resolved handles ────────────────────────────────────────────────++fn post_to_pid_fn() -> Option<PostToPidFn> {+ static SYM: OnceLock<Option<PostToPidFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"SLEventPostToPid\0").map(|p| unsafe { as_fn(p) }))+}++fn set_auth_msg_fn() -> Option<SetAuthMsgFn> {+ static SYM: OnceLock<Option<SetAuthMsgFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"SLEventSetAuthenticationMessage\0").map(|p| unsafe { as_fn(p) }))+}++fn set_window_loc_fn() -> Option<SetWindowLocFn> {+ static SYM: OnceLock<Option<SetWindowLocFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"CGEventSetWindowLocation\0").map(|p| unsafe { as_fn(p) }))+}++fn set_int_field_fn() -> Option<SetIntFieldFn> {+ static SYM: OnceLock<Option<SetIntFieldFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"SLEventSetIntegerValueField\0").map(|p| unsafe { as_fn(p) }))+}++fn connection_id_fn() -> Option<ConnectionIDFn> {+ static SYM: OnceLock<Option<ConnectionIDFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"CGSMainConnectionID\0").map(|p| unsafe { as_fn(p) }))+}++fn factory_msg_send_fn() -> Option<FactoryMsgSendFn> {+ static SYM: OnceLock<Option<FactoryMsgSendFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"objc_msgSend\0").map(|p| unsafe { as_fn(p) }))+}++fn set_front_process_fn() -> Option<SetFrontProcessFn> {+ static SYM: OnceLock<Option<SetFrontProcessFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"SLPSSetFrontProcessWithOptions\0").map(|p| unsafe { as_fn(p) }))+}++fn get_window_owner_fn() -> Option<GetWindowOwnerFn> {+ static SYM: OnceLock<Option<GetWindowOwnerFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"SLSGetWindowOwner\0").map(|p| unsafe { as_fn(p) }))+}++fn get_connection_psn_fn() -> Option<GetConnectionPSNFn> {+ static SYM: OnceLock<Option<GetConnectionPSNFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"SLSGetConnectionPSN\0").map(|p| unsafe { as_fn(p) }))+}++fn post_event_record_to_fn() -> Option<PostEventRecordToFn> {+ static SYM: OnceLock<Option<PostEventRecordToFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"SLPSPostEventRecordTo\0").map(|p| unsafe { as_fn(p) }))+}++fn get_front_process_fn() -> Option<GetFrontProcessFn> {+ static SYM: OnceLock<Option<GetFrontProcessFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"_SLPSGetFrontProcess\0").map(|p| unsafe { as_fn(p) }))+}++fn get_process_for_pid_fn() -> Option<GetProcessForPIDFn> {+ static SYM: OnceLock<Option<GetProcessForPIDFn>> = OnceLock::new();+ *SYM.get_or_init(|| find_sym(b"GetProcessForPID\0").map(|p| unsafe { as_fn(p) }))+}++/// `true` when `SLEventPostToPid` resolved.+pub fn is_available() -> bool {+ post_to_pid_fn().is_some()+}++/// `true` when all three focus-without-raise SPIs resolved.+pub fn is_focus_without_raise_available() -> bool {+ get_front_process_fn().is_some()+ && get_process_for_pid_fn().is_some()+ && post_event_record_to_fn().is_some()+}++// ── ObjC runtime helpers ───────────────────────────────────────────────────++/// Look up an ObjC class by C-string name via `objc_getClass`.+fn objc_class(name: &CStr) -> *mut c_void {+ type GetClassFn = unsafe extern "C" fn(*const c_char) -> *mut c_void;+ static SYM: OnceLock<Option<GetClassFn>> = OnceLock::new();+ let f = *SYM.get_or_init(|| find_sym(b"objc_getClass\0").map(|p| unsafe { as_fn(p) }));+ match f {+ Some(f) => unsafe { f(name.as_ptr()) },+ None => std::ptr::null_mut(),+ }+}++/// Register / look up an ObjC selector by C-string name via `sel_registerName`.+fn sel_register(name: &CStr) -> *mut c_void {+ type SelRegFn = unsafe extern "C" fn(*const c_char) -> *mut c_void;+ static SYM: OnceLock<Option<SelRegFn>> = OnceLock::new();+ let f = *SYM.get_or_init(|| find_sym(b"sel_registerName\0").map(|p| unsafe { as_fn(p) }));+ match f {+ Some(f) => unsafe { f(name.as_ptr()) },+ None => std::ptr::null_mut(),+ }+}++// ── SLSEventRecord extraction ──────────────────────────────────────────────++/// Extract the embedded `SLSEventRecord *` from a `CGEvent`.+///+/// Layout of `__CGEvent` (SkyLight ObjC type encodings):+/// `{CFRuntimeBase, uint32_t, SLSEventRecord *}`+/// On 64-bit: CFRuntimeBase=16, uint32=4, 4 bytes pad → record pointer at offset 24.+/// We probe offsets 24, 32, 16 for resilience across OS versions (same as Swift).+unsafe fn extract_event_record(event_ptr: *mut c_void) -> *mut c_void {+ for &offset in &[24usize, 32, 16] {+ let slot = (event_ptr as *const u8)+ .add(offset)+ .cast::<*mut c_void>();+ let p = std::ptr::read_unaligned(slot);+ if !p.is_null() {+ return p;+ }+ }+ std::ptr::null_mut()+}++// ── Public entry points ────────────────────────────────────────────────────++/// Post `event_ptr` (raw `CGEventRef`) to `pid` via `SLEventPostToPid`.+///+/// `attach_auth_message`: pass `true` for keyboard events (Chromium path),+/// `false` for mouse events (see Swift doc comment on `postToPid`).+///+/// Returns `true` when `SLEventPostToPid` resolved and the post was attempted.+/// Returns `false` when the SPI is absent — caller falls back to `CGEvent::post_to_pid`.+pub fn post_to_pid(pid: pid_t, event_ptr: *mut c_void, attach_auth_message: bool) -> bool {+ let post_fn = match post_to_pid_fn() {+ Some(f) => f,+ None => return false,+ };++ if attach_auth_message {+ // Build and attach SLSEventAuthenticationMessage.+ let cls = objc_class(c"SLSEventAuthenticationMessage");+ let sel = sel_register(c"messageWithEventRecord:pid:version:");+ let factory = factory_msg_send_fn();++ if !cls.is_null() && !sel.is_null() {+ if let Some(factory_fn) = factory {+ let record = unsafe { extract_event_record(event_ptr) };+ if !record.is_null() {+ let msg = unsafe { factory_fn(cls, sel, record, pid as c_int, 0u32) };+ if !msg.is_null() {+ if let Some(set_auth) = set_auth_msg_fn() {+ unsafe { set_auth(event_ptr, msg) };+ }+ }+ }+ }+ }+ }++ unsafe { post_fn(pid, event_ptr) };+ true+}++/// Stamp a window-local `(x, y)` point onto `event_ptr` via the private+/// `CGEventSetWindowLocation` SPI. Returns `true` when the SPI resolved.+pub fn set_window_location(event_ptr: *mut c_void, x: f64, y: f64) -> bool {+ match set_window_loc_fn() {+ Some(f) => { unsafe { f(event_ptr, x, y) }; true }+ None => false,+ }+}++/// Stamp `value` onto `event_ptr` at raw SkyLight field index `field` via+/// `SLEventSetIntegerValueField`. Returns `false` when SPI absent.+pub fn set_integer_field(event_ptr: *mut c_void, field: u32, value: i64) -> bool {+ match set_int_field_fn() {+ Some(f) => { unsafe { f(event_ptr, field, value) }; true }+ None => false,+ }+}++/// Return the Skylight main connection ID for the current process.+pub fn main_connection_id() -> Option<u32> {+ connection_id_fn().map(|f| unsafe { f() })+}++// ── Focus-without-raise ───────────────────────────────────────────────────────++/// Activate `target_pid`'s window `target_wid` without raising any windows+/// or triggering Space-follow. Ported from yabai's+/// `window_manager_focus_window_without_raise`.+///+/// Recipe:+/// 1. `_SLPSGetFrontProcess` → capture current front PSN.+/// 2. `GetProcessForPID(target_pid)` → target PSN.+/// 3. Post 248-byte defocus record to front PSN (`bytes[0x8a] = 0x02`).+/// 4. Post 248-byte focus record to target PSN (`bytes[0x8a] = 0x01`,+/// `bytes[0x3c..0x3f]` = `target_wid` little-endian).+///+/// Deliberately skips `SLPSSetFrontProcessWithOptions` — see the Swift+/// reference `FocusWithoutRaise.swift` for why omitting it keeps+/// Chromium's user-activation gate open.+///+/// Returns `true` when all SPIs resolved and both posts succeeded.+pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool {+ let post_fn = match post_event_record_to_fn() {+ Some(f) => f,+ None => return false,+ };+ let get_front = match get_front_process_fn() {+ Some(f) => f,+ None => return false,+ };+ let get_pid_psn = match get_process_for_pid_fn() {+ Some(f) => f,+ None => return false,+ };++ // 8-byte PSN buffers (two UInt32s).+ let mut prev_psn = [0u8; 8];+ let mut target_psn = [0u8; 8];++ let ok_prev = unsafe { get_front(prev_psn.as_mut_ptr() as *mut c_void) } == 0;+ if !ok_prev { return false; }++ let ok_target = unsafe { get_pid_psn(target_pid, target_psn.as_mut_ptr() as *mut c_void) } == 0;+ if !ok_target { return false; }++ // Build the 248-byte event buffer.+ let mut buf = [0u8; 0xF8];+ buf[0x04] = 0xF8;+ buf[0x08] = 0x0D;+ // Stamp target window id in little-endian at bytes 0x3c–0x3f.+ buf[0x3C] = (target_wid & 0xFF) as u8;+ buf[0x3D] = ((target_wid >> 8) & 0xFF) as u8;+ buf[0x3E] = ((target_wid >> 16) & 0xFF) as u8;+ buf[0x3F] = ((target_wid >> 24) & 0xFF) as u8;++ // Step 3: defocus previous front.+ buf[0x8A] = 0x02;+ let defocus_ok = unsafe {+ post_fn(prev_psn.as_ptr() as *const c_void, buf.as_ptr()) == 0+ };++ // Step 4: focus target.+ buf[0x8A] = 0x01;+ let focus_ok = unsafe {+ post_fn(target_psn.as_ptr() as *const c_void, buf.as_ptr()) == 0+ };++ defocus_ok && focus_ok+}++// ── NSMenu shortcut activation ────────────────────────────────────────────────++/// Gets the PSN for the process that owns `window_id`.+/// Uses `CGSMainConnectionID` + `SLSGetWindowOwner` + `SLSGetConnectionPSN`.+/// Falls back to `GetProcessForPID(pid)` when the SkyLight path fails.+pub fn get_process_psn_for_window(window_id: u32, pid: libc::pid_t, out_psn: &mut [u8; 8]) -> bool {+ // Try modern path: CGSMainConnectionID → SLSGetWindowOwner → SLSGetConnectionPSN+ if let (Some(get_owner), Some(get_psn), Some(conn_id_fn)) =+ (get_window_owner_fn(), get_connection_psn_fn(), connection_id_fn())+ {+ let main_cid = unsafe { conn_id_fn() };+ let mut owner_cid: u32 = 0;+ let ok = unsafe { get_owner(main_cid, window_id, &mut owner_cid) } == 0;+ if ok && owner_cid != 0 {+ let psn_ok = unsafe { get_psn(owner_cid, out_psn.as_mut_ptr() as *mut c_void) } == 0;+ if psn_ok { return true; }+ }+ }+ // Fallback: GetProcessForPID+ if let Some(get_pid_psn) = get_process_for_pid_fn() {+ return unsafe { get_pid_psn(pid, out_psn.as_mut_ptr() as *mut c_void) } == 0;+ }+ false+}++/// Activate `target_pid`'s window `target_wid` for NSMenu key dispatch, run `action`,+/// then immediately restore the prior frontmost process.+///+/// The entire activate → action → restore sequence is < 1 ms — a 5 ms UX monitor+/// never observes the intermediate frontmost state. NSMenu still fires because the+/// key event is already enqueued in the target's run-loop queue before we restore.+///+/// Returns `Ok(true)` when activation succeeded, `Ok(false)` when SPIs unavailable.+pub fn with_menu_shortcut_activation(+ target_pid: libc::pid_t,+ target_wid: u32,+ action: impl FnOnce() -> anyhow::Result<()>,+) -> anyhow::Result<bool> {+ let set_front = match set_front_process_fn() {+ Some(f) => f,+ None => {+ // SPIs unavailable — run action anyway without activation.+ action()?;+ return Ok(false);+ }+ };++ // Capture prior frontmost PSN.+ let mut prev_psn = [0u8; 8];+ let prev_ok = get_front_process_fn()+ .map(|f| unsafe { f(prev_psn.as_mut_ptr() as *mut c_void) } == 0)+ .unwrap_or(false);++ // Resolve target PSN.+ let mut target_psn = [0u8; 8];+ let target_ok = get_process_psn_for_window(target_wid, target_pid, &mut target_psn);+ if !target_ok {+ action()?;+ return Ok(false);+ }++ // Make target WindowServer-frontmost (kCPSNoWindows = 0x400).+ unsafe { set_front(target_psn.as_ptr() as *const c_void, target_wid, 0x400) };++ // Run action then restore — even if action fails.+ let result = action();++ // Restore prior frontmost (windowID=0, options=0x400).+ if prev_ok {+ unsafe { set_front(prev_psn.as_ptr() as *const c_void, 0, 0x400) };+ }++ result?;+ Ok(true)+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/lib.rs b/libs/cua-driver-rs/crates/platform-macos/src/lib.rsnew file mode 100644index 0000000000..1b1e9f8048--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/lib.rs@@ -0,0 +1,61 @@+//! macOS platform backend for cua-driver-rs.+//!+//! Provides background automation on macOS via:+//! - Accessibility (AX) API for UI tree walking and element interaction+//! - CGEvent / SkyLight SPI for background mouse and keyboard injection+//! - NSRunningApplication / NSWorkspace for app enumeration and lifecycle+//! - CGWindow / ScreenCaptureKit for window enumeration and screenshots++#[cfg(target_os = "macos")]+pub mod ax;+#[cfg(target_os = "macos")]+pub mod apps;+#[cfg(target_os = "macos")]+pub mod windows;+#[cfg(target_os = "macos")]+pub mod input;+#[cfg(target_os = "macos")]+pub mod cursor;+#[cfg(target_os = "macos")]+pub mod capture;+#[cfg(target_os = "macos")]+pub mod browser;+#[cfg(target_os = "macos")]+pub mod tools;++use mcp_server::tool::ToolRegistry;++/// Register all macOS tools. For programs that don't restructure `main`+/// (e.g. test harnesses), the overlay is skipped.+pub fn register_tools() -> ToolRegistry {+ #[cfg(target_os = "macos")]+ {+ let mut r = ToolRegistry::new();+ tools::register_all(&mut r);+ r+ }+ #[cfg(not(target_os = "macos"))]+ ToolRegistry::new()+}++/// Register all macOS tools and initialise the cursor overlay channel.+///+/// After calling this, `main()` must call+/// `platform_macos::cursor::overlay::run_on_main_thread()` on the OS+/// main thread to actually display the overlay.+pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig) -> ToolRegistry {+ #[cfg(target_os = "macos")]+ {+ if cfg.enabled {+ cursor::overlay::init(cfg);+ }+ let mut r = ToolRegistry::new();+ tools::register_all(&mut r);+ r+ }+ #[cfg(not(target_os = "macos"))]+ {+ let _ = cfg;+ ToolRegistry::new()+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/check_permissions.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/check_permissions.rsnew file mode 100644index 0000000000..40e9cde230--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/check_permissions.rs@@ -0,0 +1,109 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;++pub struct CheckPermissionsTool;++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ // Matches Swift `CheckPermissionsTool.swift` description verbatim.+ name: "check_permissions".into(),+ description: "Report TCC permission status for Accessibility and Screen Recording. \+ By default also raises the system permission dialogs for any missing grants — \+ Apple's request APIs are no-ops when the grant is already active, so this is \+ safe to call repeatedly. Pass {\"prompt\": false} for a purely read-only \+ status check.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {+ "prompt": {+ "type": "boolean",+ "description": "Raise the system permission prompts for missing grants. Default true.",+ }+ },+ "additionalProperties": false,+ }),+ // Not read_only because the default path may raise a modal dialog+ // (mirrors Swift annotation `readOnlyHint: false`).+ read_only: false,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for CheckPermissionsTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ // Default to prompting — same default + rationale as Swift.+ let should_prompt = args.get("prompt").and_then(|v| v.as_bool()).unwrap_or(true);+ if should_prompt {+ let _ = request_accessibility();+ let _ = request_screen_recording();+ }+ let accessibility = unsafe { crate::ax::bindings::AXIsProcessTrusted() };+ let screen_recording = probe_screen_recording();++ // Text format mirrors Swift 1:1:+ // "✅ Accessibility: granted.\n✅ Screen Recording: granted."+ let ax_prefix = if accessibility { "✅" } else { "❌" };+ let sr_prefix = if screen_recording { "✅" } else { "❌" };+ let ax_state = if accessibility { "granted" } else { "NOT granted" };+ let sr_state = if screen_recording { "granted" } else { "NOT granted" };+ let summary = format!(+ "{ax_prefix} Accessibility: {ax_state}.\n{sr_prefix} Screen Recording: {sr_state}."+ );++ ToolResult::text(summary)+ .with_structured(serde_json::json!({+ "accessibility": accessibility,+ "screen_recording": screen_recording,+ }))+ }+}++/// Raise the Accessibility TCC prompt if not yet granted. No-op when active.+/// Mirrors Swift `Permissions.requestAccessibility()`.+fn request_accessibility() -> bool {+ use core_foundation::base::TCFType;+ use core_foundation::boolean::CFBoolean;+ use core_foundation::dictionary::CFDictionary;+ use core_foundation::string::CFString;++ let key = CFString::new("AXTrustedCheckOptionPrompt");+ let val = CFBoolean::true_value();+ let options = CFDictionary::from_CFType_pairs(&[(key.as_CFType(), val.as_CFType())]);+ unsafe {+ crate::ax::bindings::AXIsProcessTrustedWithOptions(options.as_concrete_TypeRef())+ }+}++/// Raise the Screen Recording TCC prompt if not yet granted.+/// Mirrors Swift `Permissions.requestScreenRecording()` (`CGRequestScreenCaptureAccess`).+fn request_screen_recording() -> bool {+ #[link(name = "CoreGraphics", kind = "framework")]+ extern "C" {+ fn CGRequestScreenCaptureAccess() -> bool;+ }+ unsafe { CGRequestScreenCaptureAccess() }+}++/// Real probe — Swift uses `SCShareableContent.excludingDesktopWindows`,+/// which is unavailable from Rust without large ScreenCaptureKit bindings.+/// `CGPreflightScreenCaptureAccess` returns false negatives for+/// subprocess-launched apps but is the closest stable C API. When that+/// returns false, fall back to the existing window-enumeration heuristic+/// (returns content only when Screen Recording is actually granted).+fn probe_screen_recording() -> bool {+ #[link(name = "CoreGraphics", kind = "framework")]+ extern "C" {+ fn CGPreflightScreenCaptureAccess() -> bool;+ }+ if unsafe { CGPreflightScreenCaptureAccess() } { return true; }+ let windows = crate::windows::all_windows();+ !windows.is_empty()+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/click.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/click.rsnew file mode 100644index 0000000000..59c2a7d1d2--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/click.rs@@ -0,0 +1,397 @@+//! click tool — matches the Swift reference ClickTool.swift.+//!+//! Two addressing modes:+//!+//! * **AX path** (`element_index` + `window_id`): performs AXAction on the cached+//! element. Fires via AX RPC — the target app never needs to be frontmost.+//! Extra behaviors vs. the naive dispatch:+//! - AXTextField / AXTextArea: 800 ms post-click delay for WebKit DOM focus settle.+//! - AXPopUpButton: appends the list of available options and redirects to set_value.+//! - Advertised-action warning if the element didn't list the requested action.+//!+//! * **Pixel path** (`x`, `y`): synthesises CGEvent mouse clicks and posts them to+//! the target pid. `from_zoom=true` translates zoom-crop pixel coordinates back+//! to full-window space using the most recent `zoom` context stored per-pid.++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use crate::ax::bindings::{+ copy_action_names, copy_children, copy_string_attr, element_screen_rect,+ AXUIElementRef,+};+use core_foundation::base::CFRelease;++use super::ToolState;++pub struct ClickTool {+ state: Arc<ToolState>,+}++impl ClickTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "click".into(),+ description:+ "Left-click against a target pid. Two addressing modes:\n\n\+ - element_index + window_id (from last get_window_state): AX action path. \+ Works on backgrounded/hidden windows. No cursor move, no focus steal. \+ Preferred path.\n\n\+ - x, y (window-local screenshot pixels): CGEvent path. Synthesizes mouse \+ events and posts to pid. Use modifier for cmd/shift/option/ctrl.\n\n\+ action: press (default), show_menu, pick, confirm, cancel, open.\n\+ from_zoom: set true after a zoom call to auto-translate zoom-image pixel \+ coordinates to full-window space."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid"],+ "properties": {+ "pid": { "type": "integer", "description": "Target process ID." },+ "window_id": { "type": "integer", "description": "Target window ID. Required for element_index." },+ "element_index": { "type": "integer", "description": "Element index from last get_window_state." },+ "x": { "type": "number", "description": "Window-local screenshot X coordinate." },+ "y": { "type": "number", "description": "Window-local screenshot Y coordinate." },+ "action": { "type": "string", "description": "AX action: press, show_menu, pick, confirm, cancel, open." },+ "count": { "type": "integer", "description": "Click count (pixel path only). Default 1." },+ "modifier": {+ "type": "array",+ "items": { "type": "string" },+ "description": "Modifier keys: cmd, shift, option/alt, ctrl."+ },+ "from_zoom": {+ "type": "boolean",+ "description": "When true, x and y are in the last zoom image for this pid; driver translates back to full-window coordinates."+ },+ "debug_image_out": {+ "type": "string",+ "description": "Optional file path. When set on a pixel-addressed click, captures a fresh screenshot, draws a red crosshair at (x, y), and writes the PNG. Use to verify coordinate spaces. Requires window_id; incompatible with from_zoom."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+}++#[async_trait]+impl Tool for ClickTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };++ let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);+ let x = args.get("x").and_then(|v| v.as_f64())+ .or_else(|| args.get("x").and_then(|v| v.as_i64()).map(|i| i as f64));+ let y = args.get("y").and_then(|v| v.as_f64())+ .or_else(|| args.get("y").and_then(|v| v.as_i64()).map(|i| i as f64));+ let action = args.get("action").and_then(|v| v.as_str()).unwrap_or("press").to_owned();+ let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(1) as usize;+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);+ let debug_image_out = args.get("debug_image_out").and_then(|v| v.as_str()).map(str::to_owned);+ let modifiers: Vec<String> = args.get("modifier")+ .and_then(|v| v.as_array())+ .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();++ if let (Some(idx), Some(wid)) = (element_index, window_id) {+ // ── AX element path ────────────────────────────────────────────+ let element_ptr = match self.state.element_cache.get_element_ptr(pid, wid, idx) {+ Some(p) => p,+ None => return ToolResult::error(format!(+ "Element index {idx} not found in cache for pid={pid} window_id={wid}. \+ Call get_window_state first."+ )),+ };++ // Animate cursor to element center BEFORE firing AX action,+ // mirroring Swift's `performElementClick` → `animateAndWait(to:)`.+ let center_ptr = element_ptr;+ let center = tokio::task::spawn_blocking(move || unsafe {+ crate::ax::bindings::element_screen_center(center_ptr as AXUIElementRef)+ }).await.ok().flatten();++ if let Some((cx, cy)) = center {+ // Pin overlay above target window first.+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::PinAbove(wid as u64)+ );+ crate::cursor::overlay::animate_cursor_to(cx, cy).await;+ }++ // Run AX work on a blocking thread (can't block async executor).+ let action_clone = action.clone();+ let result = tokio::task::spawn_blocking(move || {+ perform_ax_click(element_ptr, idx, pid, wid, &action_clone)+ }).await;++ match result {+ Ok(Ok((msg, needs_webkit_delay))) => {+ // For text inputs, wait 800ms for WebKit DOM focus to settle+ // before returning — matches the Swift reference behaviour.+ if needs_webkit_delay {+ tokio::time::sleep(std::time::Duration::from_millis(800)).await;+ }+ ToolResult::text(msg)+ }+ Ok(Err(e)) => ToolResult::error(format!("AX action failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ } else if let (Some(mut cx), Some(mut cy)) = (x, y) {+ // ── Pixel path ─────────────────────────────────────────────────++ // debug_image_out: capture fresh screenshot, overlay crosshair BEFORE+ // any coordinate translation (so it shows received coords in the same+ // space the caller was reasoning in).+ if let Some(ref dbg_path) = debug_image_out {+ if from_zoom {+ return ToolResult::error(+ "debug_image_out is incompatible with from_zoom — \+ received (x, y) would be in zoom-crop space, not window-local."+ );+ }+ match window_id {+ None => return ToolResult::error(+ "debug_image_out requires window_id."+ ),+ Some(wid) => {+ let max_dim = self.state.config.read().unwrap().max_image_dimension;+ let dbg_path_c = dbg_path.clone();+ let dbg_result = tokio::task::spawn_blocking(move || {+ let png = crate::capture::screenshot_window_bytes(wid)?;+ let png = crate::capture::resize_png_if_needed(&png, max_dim)?;+ crate::capture::write_crosshair_png(&png, cx, cy, &dbg_path_c)+ }).await;+ match dbg_result {+ Err(e) => return ToolResult::error(format!(+ "debug_image_out task failed: {e}. Not dispatching click."+ )),+ Ok(Err(e)) => return ToolResult::error(format!(+ "debug_image_out write failed: {e}. Not dispatching click."+ )),+ Ok(Ok(())) => {}+ }+ }+ }+ }++ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => {+ let (wx, wy) = ctx.zoom_to_window(cx, cy);+ cx = wx;+ cy = wy;+ }+ None => return ToolResult::error(format!(+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."+ )),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ // Coordinates are in the downscaled image space; scale back to native pixels.+ cx *= ratio;+ cy *= ratio;+ }++ // ── Window-local → screen coordinate translation ──────────────────+ // `click_at_xy` accepts screen-space coordinates (top-left origin).+ // Callers supply window-local screenshot pixels; we add the window's+ // screen-origin to produce the final screen position.+ //+ // Backing scale: screencapture captures at physical pixels, so on a+ // Retina display the screenshot is 2× the logical window size.+ // We detect the scale by comparing the live screenshot dimensions to+ // the window's logical bounds from WindowServer.+ //+ // win_local_x/y: window-local logical-pixel coords (= cx/scale, cy/scale)+ // needed for CGEventSetWindowLocation in the Chromium recipe.+ let (screen_x, screen_y, win_local_x, win_local_y) = if let Some(wid) = window_id {+ let result = tokio::task::spawn_blocking(move || {+ let bounds = crate::windows::window_bounds_by_id(wid);+ let scale: f64 = if let Some(ref b) = bounds {+ // Detect Retina scale from the window screenshot.+ // We take a tiny peek at the PNG dimensions to compare+ // against the logical bounds.+ if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {+ if png.len() >= 24 {+ let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;+ let lw = b.width;+ if lw > 0.0 && pw > lw { pw / lw } else { 1.0 }+ } else { 1.0 }+ } else { 1.0 }+ } else { 1.0 };+ (bounds, scale)+ }).await.unwrap_or((None, 1.0));+ if let (Some(b), scale) = result {+ let wx = cx / scale;+ let wy = cy / scale;+ (b.x + wx, b.y + wy, wx, wy)+ } else {+ // window_id not found — fall back to treating x,y as screen coords.+ (cx, cy, cx, cy)+ }+ } else {+ // No window_id → treat x,y as screen coordinates (legacy behaviour).+ (cx, cy, cx, cy)+ };++ // Pin the overlay above the target window BEFORE animating so+ // the cursor is already sandwiched correctly while it glides in.+ if let Some(wid) = window_id {+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::PinAbove(wid as u64)+ );+ }+ // Animate the visual cursor to the click point and wait for it to+ // arrive — mirrors Swift's `AgentCursor.shared.animateAndWait(to:)`.+ crate::cursor::overlay::animate_cursor_to(screen_x, screen_y).await;+ // Show click-pulse on the agent cursor overlay.+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::ClickPulse { x: screen_x, y: screen_y }+ );++ let mods_owned = modifiers.clone();+ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();+ // When we know the window_id, pass the window-local coordinates so+ // `click_at_xy_with_window_local` can stamp `CGEventSetWindowLocation`+ // and Chromium-specific fields (f40, f51, f58, f91, f92) onto events+ // for better backgrounded-target delivery.+ if let Some(wid) = window_id {+ return crate::input::mouse::click_at_xy_with_window_local(+ pid, screen_x, screen_y,+ win_local_x, win_local_y,+ wid, count, &m,+ );+ }+ crate::input::mouse::click_at_xy(pid, screen_x, screen_y, count, &m)+ }).await;++ match result {+ Ok(Ok(())) => ToolResult::text(format!("✅ Posted click to pid {pid}.")),+ Ok(Err(e)) => ToolResult::error(format!("Click failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ } else {+ ToolResult::error(+ "Provide either (element_index + window_id) or (x + y). pid is always required."+ )+ }+ }+}++// ── AX click implementation (blocking) ───────────────────────────────────────++/// Returns `(summary_text, needs_webkit_delay)`.+fn perform_ax_click(+ element_ptr: usize,+ idx: usize,+ pid: i32,+ window_id: u32,+ action_str: &str,+) -> anyhow::Result<(String, bool)> {+ let ax_action = map_action(action_str);+ let element = element_ptr as AXUIElementRef;++ // Capture advertised actions BEFORE dispatching so we can detect silent no-ops+ // (AX returns success even when the element doesn't advertise the action).+ let advertised = unsafe { copy_action_names(element) };++ let err = unsafe { crate::ax::bindings::perform_action(element, ax_action) };+ if err != crate::ax::bindings::kAXErrorSuccess {+ anyhow::bail!("AXUIElementPerformAction({ax_action}) returned {err}");+ }++ let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default();+ let title = unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default();++ let mut summary = format!("✅ Performed {ax_action} on [{idx}] {role} \"{title}\".");++ // AXPopUpButton: list available options, redirect to set_value.+ if role == "AXPopUpButton" {+ let children = unsafe { copy_children(element) };+ if !children.is_empty() {+ let options: Vec<String> = children.iter()+ .filter_map(|&child| {+ let t = unsafe { copy_string_attr(child, "AXTitle") }.unwrap_or_default();+ let v = unsafe { copy_string_attr(child, "AXValue") }.unwrap_or_default();+ if t.is_empty() && v.is_empty() { return None; }+ Some(if v.is_empty() || v == t {+ format!("\"{t}\"")+ } else {+ format!("\"{t}\" (value: {v})")+ })+ })+ .collect();+ for &child in &children { unsafe { CFRelease(child as _); } }++ if !options.is_empty() {+ let opt_list = options.join(", ");+ summary.push_str(+ "\n\n⚠️ This is a popup/select button. The native macOS menu closes \+ immediately when the window is in the background. Do NOT use click \+ again — instead, use:\n set_value(pid, window_id, element_index, value)\n\+ Available options: ["+ );+ summary.push_str(&opt_list);+ summary.push(']');+ }+ }+ }++ // Advertised-action warning: non-fatal but surfaces likely no-ops.+ if !advertised.contains(&ax_action.to_string()) {+ let adv_list = if advertised.is_empty() { "none".into() } else { advertised.join(", ") };+ summary.push_str(&format!(+ "\n⚠️ Element does not advertise {ax_action} (actions: {adv_list}). \+ Action may have been a no-op."+ ));+ }++ // WebKit DOM focus settle: 800 ms for text inputs (returned to async caller).+ let needs_webkit_delay = ax_action == "AXPress"+ && (role == "AXTextField" || role == "AXTextArea");++ // Show focus-rect highlight around the element (matches Swift showFocusRect).+ // Also move the cursor to the element center so the glide animation plays.+ if let Some(rect) = unsafe { element_screen_rect(element) } {+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::ShowFocusRect(Some(rect))+ );+ // Animate cursor to element center.+ let cx = rect[0] + rect[2] / 2.0;+ let cy = rect[1] + rect[3] / 2.0;+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::ClickPulse { x: cx, y: cy }+ );+ }+ let _ = pid; let _ = window_id; // used by caller context++ Ok((summary, needs_webkit_delay))+}++fn map_action(action: &str) -> &'static str {+ match action.to_lowercase().as_str() {+ "press" | "click" => "AXPress",+ "show_menu" | "right_click" => "AXShowMenu",+ "pick" => "AXPick",+ "confirm" => "AXConfirm",+ "cancel" => "AXCancel",+ "open" => "AXOpen",+ _ => "AXPress",+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/cursor_tools.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/cursor_tools.rsnew file mode 100644index 0000000000..2b1f817b6c--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/cursor_tools.rs@@ -0,0 +1,449 @@+//! Agent cursor control tools: set_agent_cursor_enabled, set_agent_cursor_motion,+//! get_agent_cursor_state.+//!+//! Extended with multi-cursor customization fields matching the customer use case+//! where codex wrapper wants control over the cursor icon.++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;+use crate::cursor::state::CursorConfig;++// ── SetAgentCursorEnabled ─────────────────────────────────────────────────────++pub struct SetAgentCursorEnabledTool {+ state: Arc<ToolState>,+}++impl SetAgentCursorEnabledTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static ENABLED_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn enabled_def() -> &'static ToolDef {+ ENABLED_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_enabled".into(),+ description: "Show or hide the agent cursor overlay for a cursor instance.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["enabled"],+ "properties": {+ "enabled": { "type": "boolean", "description": "true = show, false = hide." },+ "cursor_id": { "type": "string", "description": "Cursor instance. Default: 'default'." }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for SetAgentCursorEnabledTool {+ fn def(&self) -> &ToolDef { enabled_def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let enabled = match args.get("enabled").and_then(|v| v.as_bool()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: enabled"),+ };+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ self.state.cursor_registry.set_enabled(cursor_id, enabled);+ // Drive the visual overlay.+ crate::cursor::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled));+ ToolResult::text(format!("Agent cursor '{}' {}.", cursor_id, if enabled { "enabled" } else { "disabled" }))+ }+}++// ── SetAgentCursorMotion (config) ─────────────────────────────────────────────++pub struct SetAgentCursorMotionTool {+ state: Arc<ToolState>,+}++impl SetAgentCursorMotionTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static MOTION_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn motion_def() -> &'static ToolDef {+ MOTION_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_motion".into(),+ description: "Configure the visual appearance and motion curve of an agent cursor instance.\n\n\+ Appearance (multi-cursor customization):\n\+ - cursor_id: instance name (default='default')\n\+ - cursor_icon: built-in ('arrow','crosshair','hand','dot') or PNG/SVG file path\n\+ - cursor_color: hex color e.g. '#00FFFF' or CSS name\n\+ - cursor_label: short text shown near the cursor\n\+ - cursor_size: dot radius in points (default=16)\n\+ - cursor_opacity: 0.0–1.0 (default=0.85)\n\n\+ Motion curve (Bezier path shape):\n\+ - start_handle: departure control-point fraction [0,1]. Default 0.3\n\+ - end_handle: arrival control-point fraction [0,1]. Default 0.3\n\+ - arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n\+ - arc_flow: asymmetry [-1,1]; positive bulges toward destination. Default 0.0\n\+ - spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n\+ - glide_duration_ms: flight duration per move [50,5000]. Default 160\n\+ - dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n\+ - idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {+ "cursor_id": { "type": "string", "description": "Cursor instance name. Default: 'default'." },+ "cursor_icon": { "type": "string", "description": "Built-in icon name or file path to PNG/SVG." },+ "cursor_color": { "type": "string", "description": "Hex color (e.g. '#00FFFF') or CSS color name." },+ "cursor_label": { "type": "string", "description": "Short label near the cursor dot." },+ "cursor_size": { "type": "number", "description": "Dot radius in points. Default: 16." },+ "cursor_opacity": { "type": "number", "description": "Opacity 0.0–1.0. Default: 0.85." },+ "start_handle": {+ "type": "number",+ "description": "Start-handle fraction in [0, 1]. Default 0.3."+ },+ "end_handle": {+ "type": "number",+ "description": "End-handle fraction in [0, 1]. Default 0.3."+ },+ "arc_size": {+ "type": "number",+ "description": "Arc deflection as fraction of path length [0, 1]. Default 0.25."+ },+ "arc_flow": {+ "type": "number",+ "description": "Asymmetry bias in [-1, 1]. Default 0.0."+ },+ "spring": {+ "type": "number",+ "description": "Settle damping in [0.3, 1.0]. Default 0.72."+ },+ "glide_duration_ms": {+ "type": "number",+ "minimum": 50,+ "maximum": 5000,+ "description": "Flight duration per move in ms. Default 160."+ },+ "dwell_after_click_ms": {+ "type": "number",+ "minimum": 0,+ "maximum": 5000,+ "description": "Pause after click ripple in ms. Default 80."+ },+ "idle_hide_ms": {+ "type": "number",+ "minimum": 0,+ "maximum": 60000,+ "description": "Auto-hide delay in ms. 0 = never hide. Default 20000."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for SetAgentCursorMotionTool {+ fn def(&self) -> &ToolDef { motion_def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned();++ // Start from the current state or defaults.+ let mut current = self.state.cursor_registry.get_or_create(&cursor_id);+ let config = &mut current.config;++ // ── Appearance fields ────────────────────────────────────────────────+ if let Some(icon) = args.get("cursor_icon").and_then(|v| v.as_str()) {+ config.cursor_icon = Some(icon.to_owned());+ }+ if let Some(color) = args.get("cursor_color").and_then(|v| v.as_str()) {+ config.cursor_color = Some(color.to_owned());+ }+ if let Some(label) = args.get("cursor_label").and_then(|v| v.as_str()) {+ config.cursor_label = Some(label.to_owned());+ }+ if let Some(size) = args.get("cursor_size").and_then(|v| v.as_f64()) {+ config.cursor_size = Some(size);+ }+ if let Some(opacity) = args.get("cursor_opacity").and_then(|v| v.as_f64()) {+ config.cursor_opacity = Some(opacity.clamp(0.0, 1.0));+ }++ self.state.cursor_registry.update_config(config.clone());++ // ── Motion curve fields (apply to shared overlay MotionConfig) ───────+ // JSON integers from MCP decode as i64; coerce to f64 here.+ fn num(v: Option<&Value>) -> Option<f64> {+ v.and_then(|v| v.as_f64().or_else(|| v.as_i64().map(|i| i as f64)))+ }+ let motion_changed = args.get("start_handle").is_some()+ || args.get("end_handle").is_some()+ || args.get("arc_size").is_some()+ || args.get("arc_flow").is_some()+ || args.get("spring").is_some()+ || args.get("glide_duration_ms").is_some()+ || args.get("dwell_after_click_ms").is_some()+ || args.get("idle_hide_ms").is_some();++ if motion_changed {+ // Read current motion from overlay, apply overrides, push back.+ let current_motion = crate::cursor::overlay::current_motion();+ let new_motion = current_motion.with_overrides(+ num(args.get("start_handle")),+ num(args.get("end_handle")),+ num(args.get("arc_size")),+ num(args.get("arc_flow")),+ num(args.get("spring")),+ num(args.get("glide_duration_ms")),+ num(args.get("dwell_after_click_ms")),+ num(args.get("idle_hide_ms")),+ None, // press_duration_ms not exposed+ );+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::SetMotion(new_motion.clone())+ );+ ToolResult::text(format!(+ "Cursor '{}' updated. Motion: start={:.2} end={:.2} arc={:.2} flow={:.2} \+ spring={:.2} glide={}ms dwell={}ms idle={}ms",+ cursor_id,+ new_motion.start_handle, new_motion.end_handle,+ new_motion.arc_size, new_motion.arc_flow, new_motion.spring,+ new_motion.glide_duration_ms as u32,+ new_motion.dwell_after_click_ms as u32,+ new_motion.idle_hide_ms as u32,+ )).with_structured(serde_json::to_value(¤t.config).unwrap_or_default())+ } else {+ ToolResult::text(format!("Cursor '{}' config updated.", cursor_id))+ .with_structured(serde_json::to_value(¤t.config).unwrap_or_default())+ }+ }+}++// ── SetAgentCursorStyle ───────────────────────────────────────────────────────++pub struct SetAgentCursorStyleTool {+ state: Arc<ToolState>,+}++impl SetAgentCursorStyleTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static STYLE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn style_def() -> &'static ToolDef {+ STYLE_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_style".into(),+ description:+ "Update the visual style of the agent cursor overlay.\n\n\+ - gradient_colors: array of CSS hex strings (e.g. [\"#FF0000\",\"#0000FF\"]) \+ used as the arrow fill gradient from tip to tail. Empty array reverts to \+ the default palette colours.\n\+ - bloom_color: hex string for the radial halo/bloom behind the cursor \+ (e.g. \"#00FFFF\"). Empty string reverts to the default.\n\+ - image_path: path to a PNG, JPEG, SVG, or ICO file to use as the cursor \+ icon instead of the default gradient arrow. Empty string reverts to the \+ procedural arrow.\n\+ All parameters are optional; omit any you do not want to change."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {+ "cursor_id": {+ "type": "string",+ "description": "Cursor instance. Default: 'default'."+ },+ "gradient_colors": {+ "type": "array",+ "items": { "type": "string" },+ "description": "CSS hex gradient stops tip→tail. [] = revert to default."+ },+ "bloom_color": {+ "type": "string",+ "description": "Hex bloom/halo colour (e.g. '#00FFFF'). '' = revert to default."+ },+ "image_path": {+ "type": "string",+ "description": "Path to PNG/JPEG/SVG/ICO cursor image. '' = revert to arrow."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for SetAgentCursorStyleTool {+ fn def(&self) -> &ToolDef { style_def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned();++ // ── image_path ────────────────────────────────────────────────────────+ let image_path = args.get("image_path").and_then(|v| v.as_str());+ let shape_cmd: Option<cursor_overlay::OverlayCommand> = if let Some(path) = image_path {+ if path.is_empty() {+ // Revert to procedural arrow.+ Some(cursor_overlay::OverlayCommand::SetShape(None))+ } else {+ let path_owned = path.to_owned();+ match tokio::task::spawn_blocking(move || {+ cursor_overlay::CursorShape::load(&path_owned)+ }).await {+ Ok(Ok(shape)) => {+ // Also persist to registry so it's available for state queries.+ let mut current = self.state.cursor_registry.get_or_create(&cursor_id);+ current.config.cursor_icon = Some(path.to_owned());+ self.state.cursor_registry.update_config(current.config);+ Some(cursor_overlay::OverlayCommand::SetShape(Some(shape)))+ }+ Ok(Err(e)) => return ToolResult::error(format!("Failed to load image_path: {e}")),+ Err(e) => return ToolResult::error(format!("Task error: {e}")),+ }+ }+ } else {+ None+ };++ // ── gradient_colors ───────────────────────────────────────────────────+ let gradient_colors: Vec<[u8; 4]> = if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {+ let mut out = vec![];+ for v in arr {+ if let Some(hex) = v.as_str() {+ match parse_hex_color(hex) {+ Some(c) => out.push(c),+ None => return ToolResult::error(format!("Invalid hex color: {hex}")),+ }+ }+ }+ out+ } else {+ // Not provided — don't change.+ vec![]+ };++ // ── bloom_color ───────────────────────────────────────────────────────+ let bloom_color: Option<Option<[u8; 4]>> = if let Some(hex) = args.get("bloom_color").and_then(|v| v.as_str()) {+ if hex.is_empty() {+ Some(None) // revert+ } else {+ match parse_hex_color(hex) {+ Some(c) => Some(Some(c)),+ None => return ToolResult::error(format!("Invalid bloom_color: {hex}")),+ }+ }+ } else {+ None+ };++ // ── Dispatch to overlay ───────────────────────────────────────────────+ if let Some(cmd) = shape_cmd {+ crate::cursor::overlay::send_command(cmd);+ }++ let gradient_provided = args.get("gradient_colors").is_some();+ let bloom_provided = args.get("bloom_color").is_some();+ if gradient_provided || bloom_provided {+ crate::cursor::overlay::send_command(cursor_overlay::OverlayCommand::SetGradient {+ gradient_colors,+ bloom_color: bloom_color.flatten(),+ });+ }++ // Build response summary.+ let grad_str = args.get("gradient_colors")+ .and_then(|v| v.as_array())+ .map(|arr| {+ let strs: Vec<String> = arr.iter()+ .filter_map(|v| v.as_str().map(str::to_owned))+ .collect();+ format!("[{}]", strs.join(", "))+ })+ .unwrap_or_else(|| "(unchanged)".into());++ let bloom_str = args.get("bloom_color")+ .and_then(|v| v.as_str())+ .map(|s| if s.is_empty() { "(reverted)".to_owned() } else { s.to_owned() })+ .unwrap_or_else(|| "(unchanged)".into());++ let img_str = image_path+ .map(|s| if s.is_empty() { "(reverted to arrow)".to_owned() } else { s.to_owned() })+ .unwrap_or_else(|| "(unchanged)".into());++ ToolResult::text(format!(+ "✅ cursor style: gradient_colors={grad_str} bloom_color={bloom_str} image_path={img_str}"+ ))+ }+}++/// Parse `#RRGGBB` or `#RGB` hex string to `[R, G, B, A=255]`.+fn parse_hex_color(hex: &str) -> Option<[u8; 4]> {+ let s = hex.trim_start_matches('#');+ match s.len() {+ 6 => {+ let r = u8::from_str_radix(&s[0..2], 16).ok()?;+ let g = u8::from_str_radix(&s[2..4], 16).ok()?;+ let b = u8::from_str_radix(&s[4..6], 16).ok()?;+ Some([r, g, b, 255])+ }+ 3 => {+ let r = u8::from_str_radix(&s[0..1].repeat(2), 16).ok()?;+ let g = u8::from_str_radix(&s[1..2].repeat(2), 16).ok()?;+ let b = u8::from_str_radix(&s[2..3].repeat(2), 16).ok()?;+ Some([r, g, b, 255])+ }+ _ => None,+ }+}++// ── GetAgentCursorState ───────────────────────────────────────────────────────++pub struct GetAgentCursorStateTool {+ state: Arc<ToolState>,+}++impl GetAgentCursorStateTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static STATE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn state_def() -> &'static ToolDef {+ STATE_DEF.get_or_init(|| ToolDef {+ name: "get_agent_cursor_state".into(),+ description: "Return the current state of all agent cursor instances: position, \+ config (color, icon, label, size, opacity), enabled flag.".into(),+ input_schema: serde_json::json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for GetAgentCursorStateTool {+ fn def(&self) -> &ToolDef { state_def() }++ async fn invoke(&self, _args: Value) -> ToolResult {+ let states = self.state.cursor_registry.all_states();+ let json = serde_json::to_value(&states).unwrap_or_default();+ // Top-level "enabled" mirrors Swift's field for parity — use the default cursor.+ let enabled = states.first().map(|s| s.config.enabled).unwrap_or(true);+ ToolResult::text(format!("{} cursor instance(s).", states.len()))+ .with_structured(serde_json::json!({ "cursors": json, "enabled": enabled }))+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/double_click.rsnew file mode 100644index 0000000000..c5126b0146--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/double_click.rs@@ -0,0 +1,179 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use crate::ax::bindings::{+ copy_action_names, element_screen_center, perform_action, kAXErrorSuccess,+ AXUIElementRef,+};++use super::ToolState;++pub struct DoubleClickTool {+ state: Arc<ToolState>,+}++impl DoubleClickTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "double_click".into(),+ description:+ "Double-click at (x, y) or on an AX element identified by element_index + window_id.\n\n\+ AX path (element_index provided): performs `AXOpen` when the element advertises it \+ (Finder items, openable list rows/cells); otherwise resolves the element's on-screen \+ center and falls back to a pixel double-click there.\n\n\+ Pixel path (x, y provided): two down/up pairs ~80 ms apart at the given coordinates."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid"],+ "properties": {+ "pid": { "type": "integer" },+ "x": { "type": "number", "description": "Screen X coordinate (pixel path)." },+ "y": { "type": "number", "description": "Screen Y coordinate (pixel path)." },+ "window_id": { "type": "integer", "description": "CGWindowID. Required when element_index is used." },+ "element_index": { "type": "integer", "description": "Element index from last get_window_state. Uses AX path." }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+}++#[async_trait]+impl Tool for DoubleClickTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);++ // ── AX element path ──────────────────────────────────────────────────+ if let (Some(idx), Some(wid)) = (element_index, window_id) {+ let element_ptr = match self.state.element_cache.get_element_ptr(pid, wid, idx) {+ Some(p) => p,+ None => return ToolResult::error(format!(+ "Element index {idx} not found. Call get_window_state first."+ )),+ };++ let result = tokio::task::spawn_blocking(move || {+ ax_double_click(pid, element_ptr, idx)+ }).await;++ return match result {+ Ok(Ok(msg)) => ToolResult::text(msg),+ Ok(Err(e)) => ToolResult::error(format!("double_click failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }++ // ── Pixel path ───────────────────────────────────────────────────────+ let mut cx = match args.get("x").and_then(|v| v.as_f64()) {+ Some(v) => v,+ None => return ToolResult::error(+ "Either element_index + window_id or x + y must be provided."+ ),+ };+ let mut cy = match args.get("y").and_then(|v| v.as_f64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: y"),+ };++ // Scale back from downscaled-image space to native pixels when needed.+ if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ cx *= ratio;+ cy *= ratio;+ }++ // Window-local → screen coordinate translation + win-local logical coords+ // for CGEventSetWindowLocation (matches click.rs enhancement).+ let (screen_x, screen_y, win_local_x, win_local_y) = if let Some(wid) = window_id {+ let result = tokio::task::spawn_blocking(move || {+ let bounds = crate::windows::window_bounds_by_id(wid);+ let scale: f64 = if let Some(ref b) = bounds {+ if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {+ if png.len() >= 24 {+ let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;+ let lw = b.width;+ if lw > 0.0 && pw > lw { pw / lw } else { 1.0 }+ } else { 1.0 }+ } else { 1.0 }+ } else { 1.0 };+ (bounds, scale)+ }).await.unwrap_or((None, 1.0));+ if let (Some(b), scale) = result {+ let wx = cx / scale;+ let wy = cy / scale;+ (b.x + wx, b.y + wy, wx, wy)+ } else {+ (cx, cy, cx, cy)+ }+ } else {+ (cx, cy, cx, cy)+ };++ // Pin overlay above the target window before animating.+ if let Some(wid) = window_id {+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::PinAbove(wid as u64)+ );+ }+ // Animate cursor to the click point; wait for arrival before firing.+ crate::cursor::overlay::animate_cursor_to(screen_x, screen_y).await;+ crate::cursor::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: screen_x, y: screen_y });++ let result = tokio::task::spawn_blocking(move || {+ if let Some(wid) = window_id {+ crate::input::mouse::click_at_xy_with_window_local(+ pid, screen_x, screen_y, win_local_x, win_local_y, wid, 2, &[],+ )+ } else {+ crate::input::mouse::click_at_xy(pid, screen_x, screen_y, 2, &[])+ }+ }).await;++ match result {+ Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked at ({screen_x:.1}, {screen_y:.1}).")),+ Ok(Err(e)) => ToolResult::error(format!("Double-click failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── Blocking AX path ─────────────────────────────────────────────────────────++fn ax_double_click(pid: i32, element_ptr: usize, idx: usize) -> anyhow::Result<String> {+ let element = element_ptr as AXUIElementRef;++ // Try AXOpen first (Finder items, openable list rows, document cells).+ let actions = unsafe { copy_action_names(element) };+ if actions.iter().any(|a| a == "AXOpen") {+ let err = unsafe { perform_action(element, "AXOpen") };+ if err == kAXErrorSuccess {+ return Ok(format!("AXOpen performed on element [{idx}]."));+ }+ tracing::debug!("AXOpen returned {err} for element [{idx}], falling back to pixel double-click");+ }++ // Resolve screen center and fall back to pixel double-click.+ let (cx, cy) = unsafe { element_screen_center(element) }+ .ok_or_else(|| anyhow::anyhow!("Cannot resolve screen center for element [{idx}]"))?;++ crate::cursor::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: cx, y: cy });+ crate::input::mouse::click_at_xy(pid, cx, cy, 2, &[])?;+ Ok(format!("✅ Double-clicked element [{idx}] at ({cx:.1}, {cy:.1})."))+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/drag.rsnew file mode 100644index 0000000000..1debbf2f58--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/drag.rs@@ -0,0 +1,252 @@+//! drag tool — matches the Swift reference DragTool.swift.+//!+//! Press-drag-release gesture: mouseDown at (from_x, from_y), N interpolated+//! mouseDragged events along the path, mouseUp at (to_x, to_y).+//!+//! All coordinates are in window-local screenshot pixels (same space as+//! `get_window_state` returns). `from_zoom=true` translates from the most+//! recent zoom crop context, same as click/double_click.++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use crate::input::mouse::DragButton;+use super::ToolState;++pub struct DragTool {+ pub state: Arc<ToolState>,+}++impl DragTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "drag".into(),+ description:+ "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in \+ window-local screenshot pixels — the same space get_window_state returns. \+ Top-left origin of the target's window.\n\n\+ Use for: marquee/lasso selection, drag-and-drop, resizing via a handle, \+ scrubbing a slider, repositioning a panel.\n\n\+ `duration_ms` (default 500) is the wall-clock budget for the path between \+ mouse-down and mouse-up; `steps` (default 20) is the number of intermediate \+ mouseDragged events linearly interpolated along the path. Increase both for \+ slower, more human drags; decrease for snap gestures.\n\n\+ `modifier` keys (cmd/shift/option/ctrl) are held across the entire gesture.\n\n\+ When `from_zoom` is true, coordinates are in the last zoom image for this \+ pid; the driver maps them back to window coordinates before dispatching."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "from_x", "from_y", "to_x", "to_y"],+ "properties": {+ "pid": { "type": "integer", "description": "Target process ID." },+ "window_id": {+ "type": "integer",+ "description": "CGWindowID for the window the pixel coordinates were measured against. Optional — when omitted the driver picks the frontmost window of pid."+ },+ "from_x": { "type": "number", "description": "Drag-start X in window-local screenshot pixels. Top-left origin." },+ "from_y": { "type": "number", "description": "Drag-start Y in window-local screenshot pixels. Top-left origin." },+ "to_x": { "type": "number", "description": "Drag-end X in window-local screenshot pixels." },+ "to_y": { "type": "number", "description": "Drag-end Y in window-local screenshot pixels." },+ "duration_ms": {+ "type": "integer",+ "minimum": 0,+ "maximum": 10000,+ "description": "Wall-clock duration of the drag path between mouseDown and mouseUp. Default: 500."+ },+ "steps": {+ "type": "integer",+ "minimum": 1,+ "maximum": 200,+ "description": "Number of intermediate mouseDragged events linearly interpolated along the path. Default: 20."+ },+ "modifier": {+ "type": "array",+ "items": { "type": "string" },+ "description": "Modifier keys held across the entire gesture: cmd/shift/option/ctrl."+ },+ "button": {+ "type": "string",+ "enum": ["left", "right", "middle"],+ "description": "Mouse button used for the drag. Default: left."+ },+ "from_zoom": {+ "type": "boolean",+ "description": "When true, coordinates are in the last zoom image for this pid; driver maps back to window coordinates."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+}++#[async_trait]+impl Tool for DragTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };++ // Coerce integer or float from JSON for coordinate fields.+ let coerce = |key: &str| -> Option<f64> {+ args.get(key).and_then(|v| v.as_f64())+ .or_else(|| args.get(key).and_then(|v| v.as_i64()).map(|i| i as f64))+ };++ let mut from_x = match coerce("from_x") {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: from_x"),+ };+ let mut from_y = match coerce("from_y") {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: from_y"),+ };+ let mut to_x = match coerce("to_x") {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: to_x"),+ };+ let mut to_y = match coerce("to_y") {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: to_y"),+ };++ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);+ let duration_ms = args.get("duration_ms").and_then(|v| v.as_u64()).unwrap_or(500);+ let steps = args.get("steps").and_then(|v| v.as_u64()).unwrap_or(20) as usize;+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);+ let button_str = args.get("button").and_then(|v| v.as_str()).unwrap_or("left");+ let modifiers: Vec<String> = args.get("modifier")+ .and_then(|v| v.as_array())+ .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();++ let button = match button_str.to_lowercase().as_str() {+ "left" => DragButton::Left,+ "right" => DragButton::Right,+ "middle" => DragButton::Middle,+ other => return ToolResult::error(format!(+ "Unknown button \"{other}\" — expected left, right, or middle."+ )),+ };++ // from_zoom: translate from last zoom crop context.+ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => {+ let (wx, wy) = ctx.zoom_to_window(from_x, from_y);+ let (wx2, wy2) = ctx.zoom_to_window(to_x, to_y);+ from_x = wx; from_y = wy;+ to_x = wx2; to_y = wy2;+ }+ None => return ToolResult::error(format!(+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."+ )),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ from_x *= ratio; from_y *= ratio;+ to_x *= ratio; to_y *= ratio;+ }++ // Translate window-local screenshot pixels → screen coordinates.+ // Also compute window-local logical coords for CGEventSetWindowLocation.+ let (from_sx, from_sy, from_lx, from_ly,+ to_sx, to_sy, to_lx, to_ly) = if let Some(wid) = window_id {+ let result = tokio::task::spawn_blocking(move || {+ let bounds = crate::windows::window_bounds_by_id(wid);+ let scale: f64 = if let Some(ref b) = bounds {+ if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {+ if png.len() >= 24 {+ let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;+ let lw = b.width;+ if lw > 0.0 && pw > lw { pw / lw } else { 1.0 }+ } else { 1.0 }+ } else { 1.0 }+ } else { 1.0 };+ (bounds, scale)+ }).await.unwrap_or((None, 1.0));++ if let (Some(b), scale) = result {+ let flx = from_x / scale; let fly = from_y / scale;+ let tlx = to_x / scale; let tly = to_y / scale;+ (b.x + flx, b.y + fly, flx, fly,+ b.x + tlx, b.y + tly, tlx, tly)+ } else {+ (from_x, from_y, from_x, from_y, to_x, to_y, to_x, to_y)+ }+ } else {+ (from_x, from_y, from_x, from_y, to_x, to_y, to_x, to_y)+ };++ // Animate agent cursor along drag path (start → end).+ if let Some(wid) = window_id {+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::PinAbove(wid as u64)+ );+ }+ crate::cursor::overlay::animate_cursor_to(from_sx, from_sy).await;++ // Dispatch blocking drag synthesis.+ let mods_owned = modifiers.clone();+ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();+ crate::input::mouse::drag_at_xy(+ pid,+ from_sx, from_sy,+ to_sx, to_sy,+ Some((from_lx, from_ly)),+ Some((to_lx, to_ly)),+ window_id,+ duration_ms,+ steps,+ &m,+ button,+ )+ }).await;++ // Animate cursor to end position.+ crate::cursor::overlay::animate_cursor_to(to_sx, to_sy).await;+ if let Some(wid) = window_id {+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::PinAbove(wid as u64)+ );+ }++ let mod_suffix = if modifiers.is_empty() {+ String::new()+ } else {+ format!(" with {}", modifiers.join("+"))+ };+ let btn_suffix = if button_str == "left" { String::new() } else {+ format!(" ({button_str} button)")+ };++ match result {+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Posted drag{btn_suffix}{mod_suffix} to pid {pid} \+ from window-pixel ({}, {}) → ({}, {}), \+ screen ({}, {}) → ({}, {}) \+ in {duration_ms}ms / {steps} steps.",+ from_x as i64, from_y as i64,+ to_x as i64, to_y as i64,+ from_sx as i64, from_sy as i64,+ to_sx as i64, to_sy as i64,+ )),+ Ok(Err(e)) => ToolResult::error(format!("drag failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/get_accessibility_tree.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_accessibility_tree.rsnew file mode 100644index 0000000000..27c74f9b85--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_accessibility_tree.rs@@ -0,0 +1,102 @@+//! get_accessibility_tree — lightweight desktop snapshot: running apps + visible windows.+//!+//! This is the fast, no-TCC-grant discovery tool. For the full per-window AX subtree+//! with interactive element indices, use `get_window_state` (or+//! `get_window_state` with `capture_mode="ax"` to skip the screenshot).++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct GetAccessibilityTreeTool {+ state: Arc<ToolState>,+}++impl GetAccessibilityTreeTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "get_accessibility_tree".into(),+ description:+ "Return a lightweight snapshot of the desktop: running regular apps and \+ on-screen visible windows with their bounds, z-order, and owner pid.\n\n\+ For the full AX subtree of a single window (with interactive element indices \+ you can click by), use `get_window_state` instead — that's the heavy per-window \+ tool. This one is a fast discovery read that needs no TCC grants."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {},+ "additionalProperties": false+ }),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for GetAccessibilityTreeTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, _args: Value) -> ToolResult {+ let _ = &self.state; // state not needed for this tool++ let apps = crate::apps::list_running_apps();+ let windows = crate::windows::visible_windows();++ let mut lines = vec![format!(+ "{} running app(s), {} visible window(s)",+ apps.len(), windows.len()+ )];++ for app in &apps {+ let bid = app.bundle_id.as_deref()+ .map(|b| format!(" [{b}]"))+ .unwrap_or_default();+ lines.push(format!("- {} (pid {}){}", app.name, app.pid, bid));+ }++ if !windows.is_empty() {+ lines.push(String::new());+ lines.push("Windows:".to_owned());+ for w in &windows {+ let title = if w.title.is_empty() {+ "(no title)".to_owned()+ } else {+ format!("\"{}\"", w.title)+ };+ lines.push(format!(+ "- {} (pid {}) {} [window_id: {}]",+ w.app_name, w.pid, title, w.window_id+ ));+ }+ lines.push(+ "→ Call get_window_state(pid, window_id) to inspect a window's UI.".to_owned()+ );+ }++ let structured = serde_json::json!({+ "apps": apps.iter().map(|a| serde_json::json!({+ "pid": a.pid,+ "name": a.name,+ "bundle_id": a.bundle_id+ })).collect::<Vec<_>>(),+ "windows": windows.iter().map(|w| serde_json::json!({+ "window_id": w.window_id,+ "pid": w.pid,+ "app_name": w.app_name,+ "title": w.title+ })).collect::<Vec<_>>()+ });+ ToolResult::text(lines.join("\n")).with_structured(structured)+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/get_config.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_config.rsnew file mode 100644index 0000000000..c999ff830b--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_config.rs@@ -0,0 +1,51 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct GetConfigTool {+ state: Arc<ToolState>,+}++impl GetConfigTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "get_config".into(),+ description: "Return the current cua-driver-rs configuration.".into(),+ input_schema: serde_json::json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for GetConfigTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, _args: Value) -> ToolResult {+ let cfg = self.state.config.read().unwrap();+ let cursor_enabled = self.state.cursor_registry.all_states()+ .first()+ .map(|s| s.config.enabled)+ .unwrap_or(true);+ ToolResult::text("cua-driver-rs configuration")+ .with_structured(serde_json::json!({+ "version": env!("CARGO_PKG_VERSION"),+ "platform": "macos",+ "capture_mode": cfg.capture_mode,+ "max_image_dimension": cfg.max_image_dimension,+ "agent_cursor": {+ "enabled": cursor_enabled,+ },+ }))+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/get_cursor_position.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_cursor_position.rsnew file mode 100644index 0000000000..238bd16455--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_cursor_position.rs@@ -0,0 +1,54 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;++pub struct GetCursorPositionTool;++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "get_cursor_position".into(),+ description: "Return the current mouse cursor position in screen points (origin top-left).".into(),+ input_schema: serde_json::json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for GetCursorPositionTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, _args: Value) -> ToolResult {+ // Create a null event and query its location to get cursor position.+ use core_graphics::{+ event::CGEvent,+ event_source::{CGEventSource, CGEventSourceStateID},+ };++ let result = tokio::task::spawn_blocking(|| {+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let event = CGEvent::new(source)+ .map_err(|_| anyhow::anyhow!("CGEvent::new failed"))?;+ let loc = event.location();+ Ok::<(f64, f64), anyhow::Error>((loc.x, loc.y))+ }).await;++ match result {+ Ok(Ok((x, y))) => {+ // Truncate to integers and use Swift's text format 1:1+ // (`GetCursorPositionTool.swift`: `Int(pos.x)`, `Int(pos.y)`,+ // `"✅ Cursor at (X, Y)"`).+ let (xi, yi) = (x as i64, y as i64);+ ToolResult::text(format!("✅ Cursor at ({xi}, {yi})"))+ .with_structured(serde_json::json!({ "x": xi, "y": yi }))+ }+ Ok(Err(e)) => ToolResult::error(format!("Failed to get cursor position: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/get_screen_size.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_screen_size.rsnew file mode 100644index 0000000000..f729788a44--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_screen_size.rs@@ -0,0 +1,79 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;++pub struct GetScreenSizeTool;++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ // Matches `GetScreenSizeTool.swift` description verbatim.+ name: "get_screen_size".into(),+ description: "Return the logical size of the main display in points plus its backing \+ scale factor. Agents click in points; Retina displays have scale_factor 2.0. \+ Requires no TCC permissions.".into(),+ input_schema: serde_json::json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for GetScreenSizeTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, _args: Value) -> ToolResult {+ match main_screen_size() {+ Some((w, h, scale)) => {+ // Matches Swift text format 1:1.+ ToolResult::text(format!("✅ Main display: {w}x{h} points @ {scale}x"))+ .with_structured(serde_json::json!({+ "width": w, "height": h, "scale_factor": scale,+ }))+ }+ None => ToolResult::error("No main display detected."),+ }+ }+}++/// Returns `(width_points, height_points, backing_scale_factor)` from+/// CoreGraphics — safe to call from any thread (no AppKit main-thread requirement).+///+/// The previous NSScreen-based implementation required `MainThreadMarker::new()`+/// which always returns `None` on async tokio threads, causing the tool to+/// return an error even when a display is attached.+fn main_screen_size() -> Option<(i64, i64, f64)> {+ use core_graphics::display::{CGMainDisplayID, CGDisplayBounds};++ // SAFETY: CGMainDisplayID / CGDisplayBounds are thread-safe CG APIs.+ let display_id = unsafe { CGMainDisplayID() };+ if display_id == 0 {+ return None;+ }+ let bounds = unsafe { CGDisplayBounds(display_id) };+ let w = bounds.size.width as i64;+ let h = bounds.size.height as i64;+ if w == 0 || h == 0 {+ return None;+ }++ let scale = get_backing_scale(display_id, w);+ Some((w, h, scale))+}++/// Estimate backing scale by comparing the display's pixel mode width to its+/// logical (CoreGraphics) bounds width.+fn get_backing_scale(display_id: u32, logical_w: i64) -> f64 {+ use core_graphics::display::CGDisplayPixelsWide;+ let pixel_w = unsafe { CGDisplayPixelsWide(display_id) } as i64;+ if pixel_w > 0 && logical_w > 0 {+ let ratio = pixel_w as f64 / logical_w as f64;+ // Round to nearest 0.5 to avoid floating point noise.+ (ratio * 2.0).round() / 2.0+ } else {+ 1.0+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_window_state.rsnew file mode 100644index 0000000000..f455c5ca22--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_window_state.rs@@ -0,0 +1,198 @@+use async_trait::async_trait;+use mcp_server::{protocol::{ToolResult, Content}, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct GetWindowStateTool {+ state: Arc<ToolState>,+}++impl GetWindowStateTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "get_window_state".into(),+ description: "Walk a running app's AX tree and return a Markdown rendering of its UI, \+ tagging every actionable element with [element_index N]. Pass those indices to \+ click, type_text, press_key, etc.\n\n\+ INVARIANT: call get_window_state once per turn per (pid, window_id) before any \+ element-indexed action. The index map is replaced by the next snapshot.\n\n\+ Also captures a PNG screenshot of the specified window.\n\n\+ Optional `query` filters the tree_markdown to matching lines plus their ancestor \+ chain (case-insensitive substring). The element_index values are unchanged — \+ filtering only trims the rendered Markdown.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "window_id"],+ "properties": {+ "pid": { "type": "integer", "description": "Target process ID." },+ "window_id": { "type": "integer", "description": "Target window ID from list_windows." },+ "query": { "type": "string", "description": "Case-insensitive filter for tree_markdown." },+ "capture_mode": {+ "type": "string",+ "enum": ["som", "vision", "ax"],+ "description": "som=AX+screenshot (default), vision=screenshot only (no AX walk), ax=AX only (no screenshot)."+ },+ "screenshot_out_file": {+ "type": "string",+ "description": "When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output will contain screenshot_file_path instead."+ }+ },+ "additionalProperties": false+ }),+ read_only: true,+ destructive: false,+ idempotent: false,+ open_world: false,+ })+}++#[async_trait]+impl Tool for GetWindowStateTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let window_id = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: window_id"),+ };+ let query = args.get("query").and_then(|v| v.as_str()).map(str::to_owned);+ let screenshot_out_file = args.get("screenshot_out_file").and_then(|v| v.as_str()).map(|s| {+ // Expand ~ prefix.+ if s.starts_with("~/") {+ let home = std::env::var("HOME").unwrap_or_default();+ format!("{home}/{}", &s[2..])+ } else {+ s.to_owned()+ }+ });+ let default_mode = self.state.config.read().unwrap().capture_mode.clone();+ let capture_mode = args.get("capture_mode").and_then(|v| v.as_str()).unwrap_or(&default_mode);++ // Walk AX tree (unless vision-only mode). Accept "tree" as deprecated alias for "ax".+ let capture_mode = if capture_mode == "tree" { "ax" } else { capture_mode };+ let tree_result = if capture_mode != "vision" {+ let q = query.clone();+ let result = tokio::task::spawn_blocking(move || {+ crate::ax::tree::walk_tree(pid, Some(window_id), q.as_deref())+ }).await;+ match result {+ Ok(r) => Some(r),+ Err(e) => return ToolResult::error(format!("AX tree walk failed: {e}")),+ }+ } else {+ None+ };++ // Update element cache.+ if let Some(ref r) = tree_result {+ self.state.element_cache.update(pid, window_id, &r.nodes);+ }++ // Screenshot (unless ax-only mode).+ let max_dim = self.state.config.read().unwrap().max_image_dimension;+ // Returns (b64_or_path, final_w, final_h, Option<original_w>, is_file_path)+ let screenshot = if capture_mode != "ax" {+ let out_file = screenshot_out_file.clone();+ let res = tokio::task::spawn_blocking(move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32, Option<u32>)> {+ use base64::{engine::general_purpose::STANDARD as BASE64, Engine};+ let raw = crate::capture::screenshot_window_bytes(window_id)?;+ let (orig_w, _orig_h) = crate::capture::png_dimensions(&raw)?;+ let png = crate::capture::resize_png_if_needed(&raw, max_dim)?;+ let (w, h) = crate::capture::png_dimensions(&png)?;+ let original_w = if w < orig_w { Some(orig_w) } else { None };+ if let Some(ref path) = out_file {+ std::fs::write(path, &png)?;+ Ok((None, Some(path.clone()), w, h, original_w))+ } else {+ Ok((Some(BASE64.encode(&png)), None, w, h, original_w))+ }+ }).await;+ match res {+ Ok(Ok((b64, file_path, w, h, orig_w))) => {+ // Record resize ratio so ClickTool can scale coordinates back up.+ if let Some(ow) = orig_w {+ if w > 0 {+ self.state.resize_registry.set_ratio(pid, ow as f64 / w as f64);+ }+ } else {+ self.state.resize_registry.clear_ratio(pid);+ }+ Some((b64, file_path, w, h))+ }+ Ok(Err(e)) => {+ tracing::warn!("Screenshot failed for window {window_id}: {e}");+ None+ }+ Err(e) => {+ tracing::warn!("Screenshot task error for window {window_id}: {e}");+ None+ }+ }+ } else {+ None+ };++ // Capture screenshot dimensions before consuming.+ let screenshot_dims = screenshot.as_ref().map(|(_, _, w, h)| (*w, *h));+ let screenshot_file_path = screenshot.as_ref().and_then(|(_, fp, _, _)| fp.clone());++ // Build response.+ let mut content: Vec<Content> = Vec::new();++ if let Some((b64_opt, _file_path, w, h)) = screenshot {+ if let Some(b64) = b64_opt {+ content.push(Content::image_png(b64));+ }++ // Summary text line (matching Swift reference format).+ let element_count = self.state.element_cache.element_count(pid, window_id);+ let summary = if let Some(ref r) = tree_result {+ format!(+ "window_id={window_id} pid={pid} size={}x{} elements={element_count}\n\n{}",+ w, h, r.tree_markdown+ )+ } else {+ format!("window_id={window_id} pid={pid} size={}x{}", w, h)+ };+ content.push(Content::text(summary));+ } else if let Some(ref r) = tree_result {+ let element_count = self.state.element_cache.element_count(pid, window_id);+ content.push(Content::text(format!(+ "window_id={window_id} pid={pid} elements={element_count}\n\n{}",+ r.tree_markdown+ )));+ }++ if content.is_empty() {+ return ToolResult::error("No content produced (neither AX tree nor screenshot succeeded)");+ }++ let element_count = self.state.element_cache.element_count(pid, window_id);+ let tree_md = tree_result.as_ref().map(|r| r.tree_markdown.clone()).unwrap_or_default();+ let mut structured = serde_json::json!({+ "window_id": window_id,+ "pid": pid,+ "element_count": element_count,+ "tree_markdown": tree_md+ });+ if let Some((sw, sh)) = screenshot_dims {+ structured["screenshot_width"] = serde_json::json!(sw);+ structured["screenshot_height"] = serde_json::json!(sh);+ }+ if let Some(ref fp) = screenshot_file_path {+ structured["screenshot_file_path"] = serde_json::json!(fp);+ }+ ToolResult { content, is_error: None, structured_content: Some(structured) }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/hotkey.rsnew file mode 100644index 0000000000..5c2b89819b--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/hotkey.rs@@ -0,0 +1,133 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;+use libc;++use super::ToolState;++pub struct HotkeyTool {+ state: Arc<ToolState>,+}++impl HotkeyTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "hotkey".into(),+ description:+ "Press a combination of keys simultaneously — e.g. `[\"cmd\", \"c\"]` for Copy, \+ `[\"cmd\", \"shift\", \"4\"]` for screenshot selection. The combo is posted directly \+ to the target pid's event queue; the target does NOT need to be frontmost.\n\n\+ Two delivery paths:\n\+ • Default (no window_id): auth-message envelope — Chromium/Electron apps accept \+ the keystrokes as trusted live input on macOS 14+.\n\+ • With window_id: NSMenu path — briefly activates the target WindowServer-frontmost \+ via SLPSSetFrontProcessWithOptions (kCPSNoWindows, < 1 ms), posts WITHOUT the auth \+ envelope so IOHIDPostEvent fires and NSApplication.sendEvent: dispatches NSMenu key \+ equivalents (e.g. Cmd+Z undo, Cmd+W close). Restores prior frontmost immediately. \+ Use this path when you need native menu-bar actions on non-Chromium apps.\n\n\+ Recognized modifiers: cmd/command, shift, option/alt, ctrl/control, fn. \+ Non-modifier keys use the same vocabulary as `press_key` (return, tab, escape, \+ up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, letters, \+ digits). Order: modifiers first, one non-modifier last."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "keys"],+ "properties": {+ "pid": { "type": "integer", "description": "Target process ID." },+ "keys": {+ "type": "array",+ "items": { "type": "string" },+ "minItems": 2,+ "description": "Modifier(s) and one non-modifier key, e.g. [\"cmd\", \"c\"]."+ },+ "window_id": {+ "type": "integer",+ "description": "When set, uses NSMenu path: briefly activates the window for menu key dispatch, then restores prior frontmost."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+}++/// Modifier key names — split the keys array into modifiers + base key.+fn is_modifier(k: &str) -> bool {+ matches!(+ k.to_lowercase().as_str(),+ "cmd" | "command" | "shift" | "option" | "alt" | "ctrl" | "control" | "fn"+ )+}++#[async_trait]+impl Tool for HotkeyTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let _ = &self.state;++ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };++ let raw_keys = match args.get("keys").and_then(|v| v.as_array()) {+ Some(arr) => arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect::<Vec<_>>(),+ None => return ToolResult::error("Missing required parameter: keys"),+ };++ if raw_keys.is_empty() {+ return ToolResult::error("keys must be a non-empty array of strings.");+ }++ // Split: modifiers are all entries that are modifier names; base key is everything else.+ // Typically: last non-modifier is the key; all others are modifiers.+ let modifiers: Vec<String> = raw_keys.iter()+ .filter(|k| is_modifier(k))+ .cloned()+ .collect();+ let non_modifiers: Vec<String> = raw_keys.iter()+ .filter(|k| !is_modifier(k))+ .cloned()+ .collect();++ if non_modifiers.is_empty() {+ return ToolResult::error(+ "keys must include at least one non-modifier key (e.g. \"c\" in [\"cmd\", \"c\"])."+ );+ }++ // Use the last non-modifier key; if there are multiple, treat earlier ones as extra keys.+ let key = non_modifiers.last().unwrap().clone();+ let key_display = raw_keys.join("+");+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);++ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = modifiers.iter().map(String::as_str).collect();+ if let Some(wid) = window_id {+ crate::input::skylight::with_menu_shortcut_activation(pid as libc::pid_t, wid, || {+ crate::input::keyboard::hotkey_no_auth(pid, &key, &m)+ })?;+ Ok(())+ } else {+ crate::input::keyboard::hotkey(pid, &key, &m)+ }+ }).await;++ match result {+ Ok(Ok(())) => ToolResult::text(format!("Pressed {key_display} on pid {pid}.")),+ Ok(Err(e)) => ToolResult::error(format!("hotkey failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/launch_app.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/launch_app.rsnew file mode 100644index 0000000000..f15099bc0a--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/launch_app.rs@@ -0,0 +1,275 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;++pub struct LaunchAppTool;++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "launch_app".into(),+ description:+ "Launch a macOS app in the background — the target does NOT come to the foreground.\n\n\+ Provide either `bundle_id` (preferred — unambiguous, e.g. `com.apple.calculator`) \+ or `name` (e.g. \"Calculator\"). If both are given, bundle_id wins.\n\n\+ Optional `urls` are handed to the app as open targets — for Finder, pass a folder \+ path to open a backgrounded Finder window there.\n\n\+ Optional `electron_debugging_port`: opens a Chrome DevTools Protocol (CDP) server \+ on the specified port (appends --remote-debugging-port=N to the app's argv). \+ Use this to automate Electron/VS Code/Cursor via CDP.\n\n\+ Optional `webkit_inspector_port`: opens a WebKit inspector server on the specified \+ port (sets WEBKIT_INSPECTOR_SERVER=127.0.0.1:N + TAURI_WEBVIEW_AUTOMATION=1). \+ Use this for Tauri/WebKit-based apps.\n\n\+ Optional `creates_new_application_instance`: when true, forces a new app instance \+ even if one is already running (passes -n to open).\n\n\+ Optional `additional_arguments`: extra argv strings appended after --args.\n\n\+ Returns the launched app's pid, bundle_id, name, and a `windows` array \+ (same shape as `list_windows`) so callers can skip an extra round-trip before \+ `get_window_state(pid, window_id)`."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {+ "bundle_id": {+ "type": "string",+ "description": "App bundle identifier, e.g. com.apple.calculator. Preferred over name."+ },+ "name": {+ "type": "string",+ "description": "App display name. Used only when bundle_id is absent."+ },+ "urls": {+ "type": "array",+ "items": { "type": "string" },+ "description": "Optional file paths or URLs to open with the app (e.g. a folder path for Finder)."+ },+ "electron_debugging_port": {+ "type": "integer",+ "description": "Open a Chrome DevTools Protocol server on this port (appends --remote-debugging-port=N)."+ },+ "webkit_inspector_port": {+ "type": "integer",+ "description": "Open a WebKit inspector server on this port (sets WEBKIT_INSPECTOR_SERVER env var)."+ },+ "creates_new_application_instance": {+ "type": "boolean",+ "description": "When true, force a new app instance even if already running (open -n)."+ },+ "additional_arguments": {+ "type": "array",+ "items": { "type": "string" },+ "description": "Extra arguments appended after --args when launching."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ idempotent: true,+ open_world: true,+ })+}++#[async_trait]+impl Tool for LaunchAppTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let bundle_id = args.get("bundle_id").and_then(|v| v.as_str()).map(str::to_owned);+ let name = args.get("name").and_then(|v| v.as_str()).map(str::to_owned);+ let urls: Vec<String> = args.get("urls")+ .and_then(|v| v.as_array())+ .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();+ let electron_debugging_port = args.get("electron_debugging_port").and_then(|v| v.as_u64()).map(|v| v as u16);+ let webkit_inspector_port = args.get("webkit_inspector_port").and_then(|v| v.as_u64()).map(|v| v as u16);+ let creates_new_instance = args.get("creates_new_application_instance").and_then(|v| v.as_bool()).unwrap_or(false);+ let mut additional_arguments: Vec<String> = args.get("additional_arguments")+ .and_then(|v| v.as_array())+ .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();++ if bundle_id.is_none() && name.is_none() {+ return ToolResult::error(+ "Provide either bundle_id or name to identify the app to launch."+ );+ }++ // Build additional arguments (e.g., CDP port).+ if let Some(port) = electron_debugging_port {+ additional_arguments.push(format!("--remote-debugging-port={port}"));+ }++ // Build env dict for webkit inspector.+ let mut env: std::collections::HashMap<String, String> = std::collections::HashMap::new();+ if let Some(port) = webkit_inspector_port {+ env.insert("WEBKIT_INSPECTOR_SERVER".to_string(), format!("127.0.0.1:{port}"));+ env.insert("TAURI_WEBVIEW_AUTOMATION".to_string(), "1".to_string());+ }++ let port_summary = {+ let mut s = String::new();+ if let Some(port) = electron_debugging_port {+ s.push_str(&format!("\nCDP renderer available on port {port}."));+ }+ if let Some(port) = webkit_inspector_port {+ s.push_str(&format!("\nWebKit inspector available on port {port}."));+ }+ s+ };++ // bundle_id wins when both are supplied.+ let launch_result = tokio::task::spawn_blocking(move || {+ let pid = if let Some(ref bid) = bundle_id {+ if urls.is_empty() && additional_arguments.is_empty() && env.is_empty() && !creates_new_instance {+ crate::apps::launch_app(bid)?+ } else {+ launch_with_urls_by_bundle(bid, &urls, &additional_arguments, &env, creates_new_instance)?+ }+ } else {+ let n = name.as_deref().unwrap();+ if urls.is_empty() && additional_arguments.is_empty() && env.is_empty() && !creates_new_instance {+ crate::apps::launch_app_by_name(n)?+ } else {+ launch_with_urls_by_name(n, &urls, &additional_arguments, &env, creates_new_instance)?+ }+ };++ // Retry loop: LaunchServices returns before WindowServer has+ // registered the new windows. Poll up to 5x100ms.+ let windows = resolve_windows_for_pid(pid);++ let app_info: Option<crate::apps::AppInfo> = {+ let apps = crate::apps::list_running_apps();+ apps.into_iter().find(|a| a.pid == pid)+ };++ Ok::<_, anyhow::Error>((pid, app_info, windows))+ }).await;++ match launch_result {+ Ok(Ok((pid, app_info, windows))) => {+ let app_name = app_info.as_ref().map(|a| a.name.as_str()).unwrap_or("?");+ let bid = app_info.as_ref().and_then(|a| a.bundle_id.as_deref()).unwrap_or("?");++ let mut summary = format!("Launched {app_name} (pid {pid}) in background.{port_summary}");++ if !windows.is_empty() {+ summary.push_str("\n\nWindows:");+ for w in &windows {+ let title = if w.title.is_empty() {+ "(no title)".to_owned()+ } else {+ format!("\"{}\"", w.title)+ };+ summary.push_str(&format!("\n- {title} [window_id: {}]", w.window_id));+ }+ summary.push_str(&format!(+ "\n→ Call get_window_state(pid: {pid}, window_id) to inspect."+ ));+ }++ let windows_json: Vec<Value> = windows.iter().map(|w| serde_json::json!({+ "window_id": w.window_id,+ "pid": w.pid,+ "app_name": w.app_name,+ "title": w.title,+ "bounds": {+ "x": w.bounds.x, "y": w.bounds.y,+ "width": w.bounds.width, "height": w.bounds.height+ },+ "is_on_screen": w.is_on_screen,+ })).collect();++ ToolResult::text(summary).with_structured(serde_json::json!({+ "pid": pid,+ "bundle_id": bid,+ "name": app_name,+ "windows": windows_json+ }))+ }+ Ok(Err(e)) => ToolResult::error(format!("Launch failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── Blocking helpers ──────────────────────────────────────────────────────────++/// Launch a bundle with optional URLs, extra args, env vars, and new-instance flag.+fn launch_with_urls_by_bundle(+ bundle_id: &str,+ urls: &[String],+ additional_args: &[String],+ env: &std::collections::HashMap<String, String>,+ new_instance: bool,+) -> anyhow::Result<i32> {+ let mut cmd = std::process::Command::new("open");+ if new_instance { cmd.arg("-n"); }+ cmd.args(["-g", "-b", bundle_id]);+ for url in urls { cmd.arg(url); }+ if !additional_args.is_empty() {+ cmd.arg("--args");+ for arg in additional_args { cmd.arg(arg); }+ }+ for (k, v) in env { cmd.env(k, v); }+ let status = cmd.status()?;+ if !status.success() {+ anyhow::bail!("Failed to launch {bundle_id}");+ }+ std::thread::sleep(std::time::Duration::from_millis(500));+ let apps = crate::apps::list_running_apps();+ apps.into_iter()+ .find(|a| a.bundle_id.as_deref() == Some(bundle_id))+ .map(|a| a.pid)+ .ok_or_else(|| anyhow::anyhow!("Launched {bundle_id} but could not find its pid"))+}++/// Launch by name with optional URLs, extra args, env vars, and new-instance flag.+fn launch_with_urls_by_name(+ name: &str,+ urls: &[String],+ additional_args: &[String],+ env: &std::collections::HashMap<String, String>,+ new_instance: bool,+) -> anyhow::Result<i32> {+ let mut cmd = std::process::Command::new("open");+ if new_instance { cmd.arg("-n"); }+ cmd.args(["-g", "-a", name]);+ for url in urls { cmd.arg(url); }+ if !additional_args.is_empty() {+ cmd.arg("--args");+ for arg in additional_args { cmd.arg(arg); }+ }+ for (k, v) in env { cmd.env(k, v); }+ let status = cmd.status()?;+ if !status.success() {+ anyhow::bail!("Failed to launch '{name}'");+ }+ std::thread::sleep(std::time::Duration::from_millis(500));+ let apps = crate::apps::list_running_apps();+ apps.into_iter()+ .find(|a| a.name.eq_ignore_ascii_case(name))+ .map(|a| a.pid)+ .ok_or_else(|| anyhow::anyhow!("Launched '{name}' but could not find its pid"))+}++/// Poll for the pid's layer-0 windows, retrying up to 5x100ms to absorb+/// LaunchServices → WindowServer latency (mirrors the Swift reference).+fn resolve_windows_for_pid(pid: i32) -> Vec<crate::windows::WindowInfo> {+ for attempt in 0..5 {+ let found: Vec<_> = crate::windows::all_windows()+ .into_iter()+ .filter(|w| w.pid == pid && w.layer == 0)+ .filter(|w| w.bounds.width > 1.0 && w.bounds.height > 1.0)+ .collect();+ if !found.is_empty() {+ return found;+ }+ if attempt < 4 {+ std::thread::sleep(std::time::Duration::from_millis(100));+ }+ }+ vec![]+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/list_apps.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/list_apps.rsnew file mode 100644index 0000000000..49837980e1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/list_apps.rs@@ -0,0 +1,57 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;++pub struct ListAppsTool;++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "list_apps".into(),+ // Description matches Swift `ListAppsTool.swift` verbatim.+ description: "List macOS apps — both currently running and installed-but-not-running — \+ with per-app state flags:\n\n\+ - running: is a process for this app live? (pid is 0 when false)\n\+ - active: is it the system-frontmost app? (implies running)\n\n\+ Only apps with NSApplicationActivationPolicyRegular are included — \+ background helpers and system UI agents are filtered out. Installed \+ apps come from scanning /Applications, /Applications/Utilities, \+ ~/Applications, /System/Applications, and /System/Applications/Utilities.\n\n\+ Use this for \"is X installed?\" as well as \"is X running?\". For \+ per-window state — on-screen, on-current-Space, minimized, \+ window titles — call list_windows instead. For just opening an \+ app — running or not — call launch_app({bundle_id: ...}) directly; \+ list_apps is not a prerequisite.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {},+ "additionalProperties": false+ }),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for ListAppsTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, _args: Value) -> ToolResult {+ let apps = tokio::task::spawn_blocking(crate::apps::list_all_apps).await+ .unwrap_or_default();+ let text = crate::apps::format_app_list(&apps);+ let structured = serde_json::json!({+ "apps": apps.iter().map(|a| serde_json::json!({+ "pid": a.pid,+ "name": a.name,+ "bundle_id": a.bundle_id,+ "active": a.active,+ "running": a.running+ })).collect::<Vec<_>>()+ });+ ToolResult::text(text).with_structured(structured)+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/list_windows.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/list_windows.rsnew file mode 100644index 0000000000..933942ab73--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/list_windows.rs@@ -0,0 +1,81 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;++pub struct ListWindowsTool;++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "list_windows".into(),+ description: "List all layer-0 top-level windows currently known to WindowServer. \+ Includes off-screen windows (minimized, on another Space, hidden-launched). \+ Use this to find a window_id before calling get_window_state.\n\n\+ Per-record fields: window_id, pid, app_name, title, bounds \+ (x/y/width/height, top-left origin), z_index (higher = frontmost), \+ is_on_screen, on_current_space.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {+ "pid": {+ "type": "integer",+ "description": "Optional pid filter. When set, only this pid's windows are returned."+ },+ "on_screen_only": {+ "type": "boolean",+ "description": "When true, drop windows not on the current Space. Default false."+ }+ },+ "additionalProperties": false+ }),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for ListWindowsTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid_filter: Option<i32> = args.get("pid").and_then(|v| v.as_i64()).map(|v| v as i32);+ let on_screen_only = args.get("on_screen_only").and_then(|v| v.as_bool()).unwrap_or(false);++ let mut windows = if on_screen_only {+ crate::windows::visible_windows()+ } else {+ crate::windows::all_windows()+ };++ if let Some(pid) = pid_filter {+ windows.retain(|w| w.pid == pid);+ }++ let windows_json: Vec<Value> = windows.iter().map(|w| serde_json::json!({+ "window_id": w.window_id,+ "pid": w.pid,+ "app_name": w.app_name,+ "title": w.title,+ "bounds": {+ "x": w.bounds.x,+ "y": w.bounds.y,+ "width": w.bounds.width,+ "height": w.bounds.height+ },+ "layer": w.layer,+ "z_index": w.z_index,+ "is_on_screen": w.is_on_screen,+ "on_current_space": w.on_current_space,+ "space_ids": w.space_ids,+ })).collect();++ ToolResult::text(format!("Found {} window(s).", windows_json.len()))+ .with_structured(serde_json::json!({+ "windows": windows_json,+ "current_space_id": null+ }))+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rsnew file mode 100644index 0000000000..50576531fd--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs@@ -0,0 +1,178 @@+//! MCP tool implementations for macOS.++mod list_apps;+mod list_windows;+mod get_window_state;+mod launch_app;+mod click;+mod double_click;+mod right_click;+mod drag;+mod type_text;+mod press_key;+mod hotkey;+mod set_value;+mod scroll;+mod screenshot;+mod get_screen_size;+mod get_cursor_position;+mod move_cursor;+mod cursor_tools;+mod check_permissions;+mod get_config;+mod set_config;+mod get_accessibility_tree;+mod zoom;+mod type_text_chars;+mod page;++use mcp_server::tool::ToolRegistry;+use std::sync::Arc;+use std::collections::HashMap;++use crate::{+ ax::cache::ElementCache,+ cursor::state::CursorRegistry,+};++/// Per-process zoom context — stores the padded crop origin and resize scale+/// from the most recent `zoom` call, so `click(from_zoom=true)` can translate+/// zoom-image pixel coordinates back to full-window coordinates.+#[derive(Clone, Copy, Debug)]+pub struct ZoomContext {+ /// Padded crop X origin in full-window pixel space.+ pub origin_x: f64,+ /// Padded crop Y origin in full-window pixel space.+ pub origin_y: f64,+ /// Inverse resize scale: `cw / out_w` (1.0 = no downscale).+ pub scale_inv: f64,+}++impl ZoomContext {+ /// Translate a zoom-image coordinate `(px, py)` to full-window pixel coordinates.+ pub fn zoom_to_window(&self, px: f64, py: f64) -> (f64, f64) {+ (+ self.origin_x + px * self.scale_inv,+ self.origin_y + py * self.scale_inv,+ )+ }+}++/// Thread-safe per-pid zoom context registry.+pub struct ZoomRegistry {+ inner: std::sync::Mutex<HashMap<i32, ZoomContext>>,+}++impl ZoomRegistry {+ pub fn new() -> Self { Self { inner: std::sync::Mutex::new(HashMap::new()) } }++ pub fn set(&self, pid: i32, ctx: ZoomContext) {+ self.inner.lock().unwrap().insert(pid, ctx);+ }++ pub fn get(&self, pid: i32) -> Option<ZoomContext> {+ self.inner.lock().unwrap().get(&pid).copied()+ }+}++/// Tracks the per-pid ratio applied by `max_image_dimension` downscaling.+///+/// `ratio = original_dim / resized_dim` — multiply resized image coordinates+/// by this to recover original (native) window-local pixel coordinates.+/// Mirrors Swift's `ImageResizeRegistry`.+pub struct ResizeRegistry {+ inner: std::sync::Mutex<HashMap<i32, f64>>,+}++impl ResizeRegistry {+ pub fn new() -> Self { Self { inner: std::sync::Mutex::new(HashMap::new()) } }++ /// Record that pid's screenshot was downscaled by `ratio`.+ pub fn set_ratio(&self, pid: i32, ratio: f64) {+ self.inner.lock().unwrap().insert(pid, ratio);+ }++ /// Remove the ratio entry (no active downscale).+ pub fn clear_ratio(&self, pid: i32) {+ self.inner.lock().unwrap().remove(&pid);+ }++ /// Returns the most recent ratio, or `None` if no downscale happened.+ pub fn ratio(&self, pid: i32) -> Option<f64> {+ self.inner.lock().unwrap().get(&pid).copied()+ }+}++/// Runtime-mutable driver configuration persisted across calls within a session.+pub struct DriverConfig {+ /// Default capture_mode for get_window_state when not specified per-call.+ pub capture_mode: String,+ /// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom.+ pub max_image_dimension: u32,+}++impl Default for DriverConfig {+ fn default() -> Self {+ Self {+ capture_mode: "som".to_owned(),+ max_image_dimension: 0,+ }+ }+}++/// Shared state passed to all tools.+pub struct ToolState {+ pub element_cache: Arc<ElementCache>,+ pub cursor_registry: Arc<CursorRegistry>,+ pub zoom_registry: Arc<ZoomRegistry>,+ pub resize_registry: Arc<ResizeRegistry>,+ pub config: Arc<std::sync::RwLock<DriverConfig>>,+}++impl Default for ToolState {+ fn default() -> Self {+ Self {+ element_cache: Arc::new(ElementCache::new()),+ cursor_registry: Arc::new(CursorRegistry::new()),+ zoom_registry: Arc::new(ZoomRegistry::new()),+ resize_registry: Arc::new(ResizeRegistry::new()),+ config: Arc::new(std::sync::RwLock::new(DriverConfig::default())),+ }+ }+}++/// Register all macOS tools into the registry.+pub fn register_all(registry: &mut ToolRegistry) {+ let state = Arc::new(ToolState::default());++ registry.register(Box::new(list_apps::ListAppsTool));+ registry.register(Box::new(list_windows::ListWindowsTool));+ registry.register(Box::new(get_window_state::GetWindowStateTool::new(state.clone())));+ registry.register(Box::new(launch_app::LaunchAppTool));+ registry.register(Box::new(click::ClickTool::new(state.clone())));+ registry.register(Box::new(double_click::DoubleClickTool::new(state.clone())));+ registry.register(Box::new(right_click::RightClickTool::new(state.clone())));+ registry.register(Box::new(drag::DragTool::new(state.clone())));+ registry.register(Box::new(type_text::TypeTextTool::new(state.clone())));+ registry.register(Box::new(press_key::PressKeyTool::new(state.clone())));+ registry.register(Box::new(hotkey::HotkeyTool::new(state.clone())));+ registry.register(Box::new(set_value::SetValueTool::new(state.clone())));+ registry.register(Box::new(scroll::ScrollTool::new(state.clone())));+ registry.register(Box::new(screenshot::ScreenshotTool { state: state.clone() }));+ registry.register(Box::new(get_screen_size::GetScreenSizeTool));+ registry.register(Box::new(get_cursor_position::GetCursorPositionTool));+ registry.register(Box::new(move_cursor::MoveCursorTool::new(state.clone())));+ registry.register(Box::new(cursor_tools::SetAgentCursorEnabledTool::new(state.clone())));+ registry.register(Box::new(cursor_tools::SetAgentCursorMotionTool::new(state.clone())));+ registry.register(Box::new(cursor_tools::SetAgentCursorStyleTool::new(state.clone())));+ registry.register(Box::new(cursor_tools::GetAgentCursorStateTool::new(state.clone())));+ registry.register(Box::new(check_permissions::CheckPermissionsTool));+ registry.register(Box::new(get_config::GetConfigTool::new(state.clone())));+ registry.register(Box::new(set_config::SetConfigTool::new(state.clone())));+ registry.register(Box::new(get_accessibility_tree::GetAccessibilityTreeTool::new(state.clone())));+ registry.register(Box::new(zoom::ZoomTool { state: state.clone() }));+ registry.register(Box::new(type_text_chars::TypeTextCharsTool::new(state.clone())));+ registry.register(Box::new(page::PageTool::new(state.clone())));+ // Recording / replay tools are platform-independent — live in mcp-server.+ registry.register_recording_tools();+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/move_cursor.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/move_cursor.rsnew file mode 100644index 0000000000..a3138edba8--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/move_cursor.rs@@ -0,0 +1,65 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct MoveCursorTool {+ state: Arc<ToolState>,+}++impl MoveCursorTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "move_cursor".into(),+ description: "Move the agent cursor overlay to (x, y). Does NOT move the real mouse \+ cursor — the user's cursor stays where it is. Useful for showing the agent's \+ attention without interrupting the user.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["x", "y"],+ "properties": {+ "x": { "type": "number" },+ "y": { "type": "number" },+ "cursor_id": { "type": "string", "description": "Cursor instance to move. Default: 'default'." }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for MoveCursorTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let x = match args.get("x").and_then(|v| v.as_f64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: x"),+ };+ let y = match args.get("y").and_then(|v| v.as_f64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: y"),+ };+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");++ self.state.cursor_registry.update_position(cursor_id, x, y);+ // Drive the visual overlay (no-op when overlay is disabled).+ // End pointing upper-left (45°) — matches Swift's+ // `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention.+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::MoveTo { x, y, end_heading_radians: std::f64::consts::FRAC_PI_4 }+ );+ ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1})."))+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/page.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/page.rsnew file mode 100644index 0000000000..15417f4c8f--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/page.rs@@ -0,0 +1,296 @@+//! PageTool: MCP tool for browser JS execution, text extraction, and DOM querying.++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;+use crate::browser::{BrowserJs, ElectronJs, is_wk_web_view_app, AXPageReader};++pub struct PageTool {+ pub state: Arc<ToolState>,+}++impl PageTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "page".into(),+ description: "Interact with the browser page loaded in a running app. Supports \+ Chrome, Brave, Edge, Safari (via AppleScript), Electron apps (via CDP), \+ and WKWebView/Tauri apps (via AX tree fallback).\n\n\+ Actions:\n\+ - execute_javascript: Run JS and return the result.\n\+ - get_text: Extract visible text from the page.\n\+ - query_dom: Find elements matching a CSS selector.\n\+ - enable_javascript_apple_events: Patch Preferences to allow JS from Apple Events \+ (Chrome/Brave/Edge only, requires user confirmation and browser restart).".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "window_id", "action"],+ "properties": {+ "pid": { "type": "integer", "description": "Target process ID." },+ "window_id": { "type": "integer", "description": "Target window ID from list_windows." },+ "action": {+ "type": "string",+ "enum": ["execute_javascript", "get_text", "query_dom", "enable_javascript_apple_events"],+ "description": "Action to perform."+ },+ "javascript": {+ "type": "string",+ "description": "JavaScript to execute. Required for execute_javascript."+ },+ "css_selector": {+ "type": "string",+ "description": "CSS selector for query_dom (e.g. 'a', 'button', 'input', 'h1'-'h6', 'p', 'img', 'select', '*')."+ },+ "attributes": {+ "type": "array",+ "items": { "type": "string" },+ "description": "Element attributes to include in query_dom results."+ },+ "bundle_id": {+ "type": "string",+ "description": "Bundle ID of the browser. Required for enable_javascript_apple_events."+ },+ "user_has_confirmed_enabling": {+ "type": "boolean",+ "description": "Must be true to proceed with enable_javascript_apple_events. \+ This will quit and relaunch the browser."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ idempotent: false,+ open_world: false,+ })+}++#[async_trait]+impl Tool for PageTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let window_id = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: window_id"),+ };+ let action = match args.get("action").and_then(|v| v.as_str()) {+ Some(v) => v.to_owned(),+ None => return ToolResult::error("Missing required parameter: action"),+ };++ // Resolve bundle_id from pid.+ let bundle_id = {+ let p = pid;+ tokio::task::spawn_blocking(move || {+ crate::apps::list_running_apps()+ .into_iter()+ .find(|a| a.pid == p)+ .and_then(|a| a.bundle_id)+ .unwrap_or_default()+ }).await.unwrap_or_default()+ };++ match action.as_str() {+ "enable_javascript_apple_events" => {+ let confirmed = args.get("user_has_confirmed_enabling")+ .and_then(|v| v.as_bool())+ .unwrap_or(false);+ if !confirmed {+ return ToolResult::error(+ "Set user_has_confirmed_enabling=true to proceed. \+ This will quit and relaunch the browser."+ );+ }+ let bid = args.get("bundle_id")+ .and_then(|v| v.as_str())+ .unwrap_or(&bundle_id)+ .to_owned();+ if bid.is_empty() {+ return ToolResult::error("bundle_id is required for enable_javascript_apple_events");+ }+ match BrowserJs::enable_javascript_apple_events(&bid).await {+ Ok(()) => ToolResult::text("JavaScript from Apple Events has been enabled. The browser is restarting."),+ Err(e) => ToolResult::error(format!("Failed to enable JavaScript Apple Events: {e}")),+ }+ }++ "execute_javascript" => {+ let js = match args.get("javascript").and_then(|v| v.as_str()) {+ Some(v) => v.to_owned(),+ None => return ToolResult::error("Missing required parameter: javascript"),+ };+ match execute_js(&js, &bundle_id, pid, window_id).await {+ Ok(result) => ToolResult::text(result),+ Err(e) => ToolResult::error(format!("{e}")),+ }+ }++ "get_text" => {+ let use_ax_fallback = !BrowserJs::supports(&bundle_id) && {+ let p = pid;+ tokio::task::spawn_blocking(move || is_wk_web_view_app(p)).await.unwrap_or(false)+ };++ if use_ax_fallback {+ match ax_text_fallback(pid, window_id).await {+ Ok(text) => ToolResult::text(text),+ Err(e) => ToolResult::error(format!("AX text extraction failed: {e}")),+ }+ } else {+ match execute_js("document.body.innerText", &bundle_id, pid, window_id).await {+ Ok(result) => ToolResult::text(result),+ Err(_) => {+ // Try AX fallback on error.+ match ax_text_fallback(pid, window_id).await {+ Ok(text) => ToolResult::text(text),+ Err(e) => ToolResult::error(format!("Page text extraction failed: {e}")),+ }+ }+ }+ }+ }++ "query_dom" => {+ let selector = args.get("css_selector")+ .and_then(|v| v.as_str())+ .unwrap_or("*")+ .to_owned();+ let attributes: Vec<String> = args.get("attributes")+ .and_then(|v| v.as_array())+ .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();++ let use_ax_fallback = !BrowserJs::supports(&bundle_id) && {+ let p = pid;+ tokio::task::spawn_blocking(move || is_wk_web_view_app(p)).await.unwrap_or(false)+ };++ if use_ax_fallback {+ match ax_query_fallback(pid, window_id, &selector).await {+ Ok(results) => ToolResult::text(format_ax_elements(&results)),+ Err(e) => ToolResult::error(format!("AX DOM query failed: {e}")),+ }+ } else {+ let js = build_query_selector_js(&selector, &attributes);+ match execute_js(&js, &bundle_id, pid, window_id).await {+ Ok(result) => ToolResult::text(result),+ Err(_) => {+ // Try AX fallback.+ match ax_query_fallback(pid, window_id, &selector).await {+ Ok(results) => ToolResult::text(format_ax_elements(&results)),+ Err(e) => ToolResult::error(format!("DOM query failed: {e}")),+ }+ }+ }+ }+ }++ other => ToolResult::error(format!("Unknown action: {other}")),+ }+ }+}++/// Route JavaScript execution to the appropriate backend.+async fn execute_js(js: &str, bundle_id: &str, pid: i32, window_id: u32) -> anyhow::Result<String> {+ if BrowserJs::supports(bundle_id) {+ BrowserJs::execute(js, bundle_id, window_id).await+ } else {+ let p = pid;+ let is_electron = tokio::task::spawn_blocking(move || ElectronJs::is_electron(p)).await?;+ if is_electron {+ ElectronJs::execute(js, pid).await+ } else {+ let p = pid;+ let is_wk = tokio::task::spawn_blocking(move || is_wk_web_view_app(p)).await?;+ if is_wk {+ anyhow::bail!(+ "execute_javascript is not available for WKWebView/Tauri apps. \+ Use get_text or query_dom instead."+ );+ } else {+ anyhow::bail!("Unsupported browser: bundle_id={bundle_id}");+ }+ }+ }+}++/// Extract page text via the AX tree.+async fn ax_text_fallback(pid: i32, window_id: u32) -> anyhow::Result<String> {+ let result = tokio::task::spawn_blocking(move || {+ crate::ax::tree::walk_tree(pid, Some(window_id), None)+ }).await+ .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;+ Ok(AXPageReader::extract_text(&result.tree_markdown))+}++/// Query AX tree by CSS selector.+async fn ax_query_fallback(pid: i32, window_id: u32, selector: &str) -> anyhow::Result<Vec<crate::browser::ax_page_reader::AXElement>> {+ let sel = selector.to_owned();+ let result = tokio::task::spawn_blocking(move || {+ crate::ax::tree::walk_tree(pid, Some(window_id), None)+ }).await+ .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;+ Ok(AXPageReader::query(&sel, &result.tree_markdown))+}++fn format_ax_elements(elements: &[crate::browser::ax_page_reader::AXElement]) -> String {+ if elements.is_empty() {+ return "No elements found.".to_owned();+ }+ let mut lines = Vec::new();+ for el in elements {+ let index_str = el.index.map(|i| format!("[{i}] ")).unwrap_or_default();+ let title = if !el.title.is_empty() { format!(" \"{}\"", el.title) } else { String::new() };+ let value = if !el.value.is_empty() { format!(" = \"{}\"", el.value) } else { String::new() };+ let desc = if !el.description.is_empty() { format!(" ({})", el.description) } else { String::new() };+ lines.push(format!("- {index_str}{}{title}{value}{desc}", el.role));+ }+ lines.join("\n")+}++/// Build a querySelectorAll JS snippet.+fn build_query_selector_js(selector: &str, attributes: &[String]) -> String {+ let escaped_sel = json_string(selector);+ let attrs_js = if attributes.is_empty() {+ "['tagName','textContent','href','id','class','type','value','placeholder','name']".to_owned()+ } else {+ let parts: Vec<String> = attributes.iter().map(|a| json_string(a)).collect();+ format!("[{}]", parts.join(","))+ };+ format!(+ r#"(function(){{+ var sel = {escaped_sel};+ var attrs = {attrs_js};+ var els = Array.from(document.querySelectorAll(sel));+ return JSON.stringify(els.map(function(el){{+ var obj = {{}};+ attrs.forEach(function(a){{ obj[a] = el[a] !== undefined ? el[a] : el.getAttribute(a); }});+ obj.textContent = (el.textContent||'').trim().substring(0,200);+ return obj;+ }}));+}})();"#+ )+}++fn json_string(value: &str) -> String {+ let escaped = value+ .replace('\\', "\\\\")+ .replace('"', "\\\"")+ .replace('\n', "\\n")+ .replace('\r', "\\r")+ .replace('\t', "\\t");+ format!("\"{escaped}\"")+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/press_key.rsnew file mode 100644index 0000000000..0a3b8df3e7--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/press_key.rs@@ -0,0 +1,119 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;+use libc;++use super::ToolState;++pub struct PressKeyTool {+ state: Arc<ToolState>,+}++impl PressKeyTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "press_key".into(),+ description: "Press and release a single key, delivered to the target pid via \+ CGEventPostToPid. No focus steal.\n\n\+ Two delivery paths:\n\+ • window_id + element_index: focuses the AX element first, then posts via the \+ auth-message path (Chromium-safe).\n\+ • window_id only (no element_index): NSMenu path — briefly activates the window \+ WindowServer-frontmost via SLPSSetFrontProcessWithOptions (kCPSNoWindows, < 1 ms), \+ posts WITHOUT the auth envelope so IOHIDPostEvent fires and NSApplication.sendEvent: \+ dispatches NSMenu key equivalents. Restores prior frontmost immediately.\n\+ • No window_id: standard auth-message path.\n\n\+ Key names: return, tab, escape, up/down/left/right, space, delete, home, end, \+ pageup, pagedown, f1-f12, plus any letter or digit.\n\+ Modifiers array: cmd, shift, option/alt, ctrl, fn.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "key"],+ "properties": {+ "pid": { "type": "integer" },+ "key": { "type": "string", "description": "Key name: return, tab, escape, up, down, etc." },+ "modifiers": {+ "type": "array",+ "items": { "type": "string" },+ "description": "Modifier keys: cmd, shift, option/alt, ctrl, fn."+ },+ "window_id": { "type": "integer" },+ "element_index": { "type": "integer" }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+}++#[async_trait]+impl Tool for PressKeyTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let key_raw = match args.get("key").and_then(|v| v.as_str()) {+ Some(v) => v.to_owned(),+ None => return ToolResult::error("Missing required parameter: key"),+ };+ let mut modifiers: Vec<String> = args.get("modifiers")+ .and_then(|v| v.as_array())+ .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();+ let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);++ // Remap "+" / "plus" → "=" + Shift (same physical key on US layout).+ let key = if key_raw == "+" || key_raw == "plus" {+ if !modifiers.iter().any(|m| m.eq_ignore_ascii_case("shift")) {+ modifiers.push("shift".to_string());+ }+ "=".to_string()+ } else {+ key_raw.clone()+ };+ let display_key = key_raw.clone();++ // Pre-focus element if requested (element_index + window_id path).+ if let (Some(idx), Some(wid)) = (element_index, window_id) {+ if let Some(element_ptr) = self.state.element_cache.get_element_ptr(pid, wid, idx) {+ let _ = tokio::task::spawn_blocking(move || {+ crate::input::ax_actions::focus_element(element_ptr)+ }).await;+ tokio::time::sleep(std::time::Duration::from_millis(30)).await;+ }+ }++ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = modifiers.iter().map(String::as_str).collect();+ if let Some(wid) = window_id {+ if element_index.is_none() {+ // NSMenu path: window_id set but no element_index.+ crate::input::skylight::with_menu_shortcut_activation(pid as libc::pid_t, wid, || {+ crate::input::keyboard::press_key_no_auth(pid, &key, &m)+ })?;+ return Ok(());+ }+ }+ crate::input::keyboard::press_key(pid, &key, &m)+ }).await;++ match result {+ Ok(Ok(())) => ToolResult::text(format!("✅ Pressed {display_key} on pid {pid}.")),+ Ok(Err(e)) => ToolResult::error(format!("press_key failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/right_click.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/right_click.rsnew file mode 100644index 0000000000..4b515ad754--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/right_click.rs@@ -0,0 +1,230 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use crate::ax::bindings::{+ copy_action_names, perform_action, kAXErrorSuccess, AXUIElementRef,+ copy_string_attr,+};++use super::ToolState;++pub struct RightClickTool {+ state: Arc<ToolState>,+}++impl RightClickTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "right_click".into(),+ description:+ "Right-click against a target pid. Two addressing modes:\n\n\+ - `element_index` + `window_id` (from the last `get_window_state` snapshot) — \+ performs `AXShowMenu` on the cached element. Pure AX RPC, works on backgrounded / \+ hidden windows, no cursor move or focus steal. Requires a prior \+ `get_window_state(pid, window_id)` in this turn.\n\n\+ - `x`, `y` — synthesizes `rightMouseDown` / `rightMouseUp` CGEvent pair posted \+ to the pid. Driver converts image-pixel → screen-point internally. \+ `modifier` forces the CGEvent path (AX actions don't propagate modifier keys).\n\n\+ Exactly one of `element_index` or (`x` AND `y`) must be provided. `pid` always \+ required. `window_id` required when `element_index` is used."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid"],+ "properties": {+ "pid": { "type": "integer", "description": "Target process ID." },+ "element_index": {+ "type": "integer",+ "description": "Element index from last get_window_state. Routes through AXShowMenu. Requires window_id."+ },+ "window_id": {+ "type": "integer",+ "description": "CGWindowID. Required when element_index is used."+ },+ "x": {+ "type": "number",+ "description": "X in window-local screenshot pixels. Must be provided together with y."+ },+ "y": {+ "type": "number",+ "description": "Y in window-local screenshot pixels. Must be provided together with x."+ },+ "modifier": {+ "type": "array",+ "items": { "type": "string" },+ "description": "Modifier keys held during the right-click: cmd/shift/option/ctrl. Pixel path only."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+}++#[async_trait]+impl Tool for RightClickTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };++ let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);+ let x = args.get("x").and_then(|v| v.as_f64());+ let y = args.get("y").and_then(|v| v.as_f64());+ let has_xy = x.is_some() && y.is_some();+ let partial_xy = x.is_some() != y.is_some();+ let modifiers: Vec<String> = args.get("modifier")+ .and_then(|v| v.as_array())+ .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();++ if partial_xy {+ return ToolResult::error("Provide both x and y together, not just one.");+ }+ if element_index.is_some() && has_xy {+ return ToolResult::error("Provide either element_index or (x, y), not both.");+ }+ if element_index.is_none() && !has_xy {+ return ToolResult::error(+ "Provide element_index or (x, y) to address the right-click target."+ );+ }+ if element_index.is_some() && window_id.is_none() {+ return ToolResult::error(+ "window_id is required when element_index is used."+ );+ }++ // ── AX element path ──────────────────────────────────────────────────+ if let (Some(idx), Some(wid)) = (element_index, window_id) {+ let element_ptr = match self.state.element_cache.get_element_ptr(pid, wid, idx) {+ Some(p) => p,+ None => return ToolResult::error(format!(+ "Element index {idx} not found. Call get_window_state first."+ )),+ };++ let result = tokio::task::spawn_blocking(move || {+ ax_show_menu(element_ptr, idx)+ }).await;++ return match result {+ Ok(Ok(msg)) => ToolResult::text(msg),+ Ok(Err(e)) => ToolResult::error(format!("AXShowMenu failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }++ // ── Pixel path ───────────────────────────────────────────────────────+ let (mut cx, mut cy) = (x.unwrap(), y.unwrap());+ // Scale back from downscaled-image space to native pixels when needed.+ if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ cx *= ratio;+ cy *= ratio;+ }++ // Window-local → screen coordinate translation + win-local logical coords+ // for CGEventSetWindowLocation (matches click.rs enhancement).+ let (screen_x, screen_y, win_local_x, win_local_y) = if let Some(wid) = window_id {+ let result = tokio::task::spawn_blocking(move || {+ let bounds = crate::windows::window_bounds_by_id(wid);+ let scale: f64 = if let Some(ref b) = bounds {+ if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {+ if png.len() >= 24 {+ let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;+ let lw = b.width;+ if lw > 0.0 && pw > lw { pw / lw } else { 1.0 }+ } else { 1.0 }+ } else { 1.0 }+ } else { 1.0 };+ (bounds, scale)+ }).await.unwrap_or((None, 1.0));+ if let (Some(b), scale) = result {+ let wx = cx / scale;+ let wy = cy / scale;+ (b.x + wx, b.y + wy, wx, wy)+ } else {+ (cx, cy, cx, cy)+ }+ } else {+ (cx, cy, cx, cy)+ };++ // Pin overlay above the target window before animating.+ if let Some(wid) = window_id {+ crate::cursor::overlay::send_command(+ cursor_overlay::OverlayCommand::PinAbove(wid as u64)+ );+ }+ // Animate cursor to the click point; wait for arrival before firing.+ crate::cursor::overlay::animate_cursor_to(screen_x, screen_y).await;+ crate::cursor::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: screen_x, y: screen_y });++ let mod_suffix = if modifiers.is_empty() {+ String::new()+ } else {+ format!(" with {}", modifiers.join("+"))+ };++ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = modifiers.iter().map(String::as_str).collect();+ if window_id.is_some() {+ crate::input::mouse::right_click_at_xy_with_window_local(+ pid, screen_x, screen_y, win_local_x, win_local_y, &m,+ )+ } else {+ crate::input::mouse::right_click_at_xy(pid, screen_x, screen_y, &m)+ }+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("Right-clicked{mod_suffix} at ({screen_x:.1}, {screen_y:.1}).")),+ Ok(Err(e)) => ToolResult::error(format!("Right-click failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── Blocking AX path ─────────────────────────────────────────────────────────++fn ax_show_menu(element_ptr: usize, idx: usize) -> anyhow::Result<String> {+ let element = element_ptr as AXUIElementRef;++ let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default();+ let title = unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default();++ let advertised = unsafe { copy_action_names(element) };+ let err = unsafe { perform_action(element, "AXShowMenu") };++ if err != kAXErrorSuccess {+ anyhow::bail!("AXUIElementPerformAction(AXShowMenu) returned {err}");+ }++ let mut summary = format!("Shown menu for [{idx}] {role} \"{title}\".");+ if !advertised.iter().any(|a| a == "AXShowMenu") {+ let list = if advertised.is_empty() {+ "none".to_owned()+ } else {+ advertised.join(", ")+ };+ summary.push_str(&format!(+ "\n⚠️ Element does not advertise AXShowMenu (actions: {list}). \+ Action may have been a no-op. Retry with a pixel right-click \+ (right_click(pid, x, y)) if no menu appeared."+ ));+ }+ Ok(summary)+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rsnew file mode 100644index 0000000000..eef822ae52--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs@@ -0,0 +1,138 @@+use async_trait::async_trait;+use mcp_server::{protocol::{ToolResult, Content}, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct ScreenshotTool {+ pub state: Arc<ToolState>,+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "screenshot".into(),+ description:+ "Capture a screenshot. Returns base64-encoded image data in the requested format \+ (default png).\n\n\+ Without `window_id`, captures the full main display. With `window_id`, captures \+ just that window (pair with `list_windows` or `get_accessibility_tree` which return \+ window IDs).\n\n\+ Requires the Screen Recording TCC grant — call `check_permissions` first if unsure."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {+ "window_id": {+ "type": "integer",+ "description": "Optional CGWindowID to capture just that window. Without it, captures the full main display."+ },+ "format": {+ "type": "string",+ "enum": ["png", "jpeg"],+ "description": "Image format. Default: png."+ },+ "quality": {+ "type": "integer",+ "minimum": 1,+ "maximum": 95,+ "description": "JPEG quality 1-95; ignored for png. Default: 95."+ }+ },+ "additionalProperties": false+ }),+ read_only: true,+ destructive: false,+ idempotent: false,+ open_world: false,+ })+}++#[async_trait]+impl Tool for ScreenshotTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();+ let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8;+ let use_jpeg = format == "jpeg";+ let max_dim = self.state.config.read().unwrap().max_image_dimension;++ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(String, u32, u32, bool)> {+ let png_bytes = if let Some(wid) = window_id {+ crate::capture::screenshot_window_bytes(wid)?+ } else {+ crate::capture::screenshot_display_bytes()?+ };+ let png_bytes = crate::capture::resize_png_if_needed(&png_bytes, max_dim)?;+ let (w, h) = crate::capture::png_dimensions(&png_bytes)?;+ if use_jpeg {+ let jpeg = crate::capture::png_bytes_to_jpeg(&png_bytes, quality)?;+ let b64 = base64_encode(&jpeg);+ Ok((b64, w, h, true))+ } else {+ let b64 = base64_encode(&png_bytes);+ Ok((b64, w, h, false))+ }+ }).await;++ match result {+ Ok(Ok((b64, w, h, is_jpeg))) => {+ let format_str = if is_jpeg { "jpeg" } else { "png" };+ let target_str = if let Some(wid) = window_id {+ format!("window {wid}")+ } else {+ "main display".to_owned()+ };++ // List visible windows (matching Swift reference).+ let mut summary_lines = vec![+ format!("Screenshot — {w}x{h} {format_str} ({target_str})")+ ];+ let visible = crate::windows::visible_windows();+ if !visible.is_empty() {+ summary_lines.push("\nOn-screen windows:".to_owned());+ for w_info in &visible {+ let title = if w_info.title.is_empty() {+ "(no title)".to_owned()+ } else {+ format!("\"{}\"", w_info.title)+ };+ summary_lines.push(format!(+ "- {} (pid {}) {} [window_id: {}]",+ w_info.app_name, w_info.pid, title, w_info.window_id+ ));+ }+ summary_lines.push(+ "→ Call get_window_state(pid, window_id) to inspect a window's UI.".to_owned()+ );+ }+ let summary = summary_lines.join("\n");++ let image_content = if is_jpeg {+ Content::image_jpeg(b64)+ } else {+ Content::image_png(b64)+ };++ ToolResult {+ content: vec![image_content, Content::text(summary)],+ is_error: None,+ structured_content: Some(serde_json::json!({+ "width": w, "height": h, "format": format_str+ })),+ }+ }+ Ok(Err(e)) => ToolResult::error(format!("Screenshot failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++fn base64_encode(data: &[u8]) -> String {+ use base64::{engine::general_purpose::STANDARD as BASE64, Engine};+ BASE64.encode(data)+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot_compat.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot_compat.rsnew file mode 100644index 0000000000..4468996a38--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot_compat.rs@@ -0,0 +1,132 @@+//! Claude Code computer-use compatibility screenshot tool.+//!+//! Replaces the regular `screenshot` tool in `--claude-code-computer-use-compat` mode.+//! Differences from the regular tool:+//! - `pid` and `window_id` are both required (regular tool makes window_id optional).+//! - Returns JPEG at 85% quality (Claude Code vision flows prefer smaller images).+//! - Validates that the target window is on-screen and layer-0 before capturing.+//! - Includes a short text note directing the caller to use pixel-addressed tools.++use async_trait::async_trait;+use mcp_server::{protocol::{ToolResult, Content}, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct ClaudeCodeCompatScreenshotTool {+ pub state: Arc<ToolState>,+}++impl ClaudeCodeCompatScreenshotTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "screenshot".into(),+ description:+ "Capture a target window and return a JPEG image. Coordinates accepted by \+ CuaDriver's pixel tools are pixels in this window screenshot's coordinate space.\n\n\+ This is the compatibility anchor for Claude Code vision flows: CuaDriver remains \+ window-scoped, and all other tools are the normal CuaDriver tools."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "window_id"],+ "properties": {+ "pid": {+ "type": "integer",+ "description": "Target process ID from list_windows or launch_app."+ },+ "window_id": {+ "type": "integer",+ "description": "Target CGWindowID from list_windows or launch_app."+ }+ },+ "additionalProperties": false+ }),+ read_only: true,+ destructive: false,+ idempotent: false,+ open_world: false,+ })+}++#[async_trait]+impl Tool for ClaudeCodeCompatScreenshotTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let window_id = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: window_id"),+ };++ // Validate: window must be visible and layer-0.+ let window = {+ let wid = window_id;+ let p = pid;+ tokio::task::spawn_blocking(move || {+ crate::windows::all_windows()+ .into_iter()+ .find(|w| w.window_id == wid && w.pid == p && w.layer == 0+ && w.is_on_screen+ && w.bounds.width > 1.0+ && w.bounds.height > 1.0)+ }).await.unwrap_or(None)+ };++ let window = match window {+ Some(w) => w,+ None => return ToolResult::error(format!(+ "No visible layer-0 window {window_id} found for pid {pid}. \+ Use list_windows to choose an on-screen target window."+ )),+ };++ let max_dim = self.state.config.read().unwrap().max_image_dimension;++ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(String, u32, u32)> {+ let png_bytes = crate::capture::screenshot_window_bytes(window_id)?;+ let png_bytes = crate::capture::resize_png_if_needed(&png_bytes, max_dim)?;+ let (w, h) = crate::capture::png_dimensions(&png_bytes)?;+ let jpeg = crate::capture::png_bytes_to_jpeg(&png_bytes, 85)?;+ use base64::{engine::general_purpose::STANDARD as BASE64, Engine};+ Ok((BASE64.encode(&jpeg), w, h))+ }).await;++ match result {+ Ok(Ok((b64, w, h))) => {+ let app_name = window.app_name;+ let summary = format!(+ "Captured window screenshot {w}x{h} for {app_name} \+ [pid: {pid}, window_id: {window_id}]. \+ Use CuaDriver pixel tools with this window-local coordinate space."+ );+ ToolResult {+ content: vec![+ Content::image_jpeg(b64),+ Content::text(summary),+ ],+ is_error: None,+ structured_content: Some(serde_json::json!({+ "pid": pid,+ "window_id": window_id,+ "width": w,+ "height": h,+ "format": "jpeg"+ })),+ }+ }+ Ok(Err(e)) => ToolResult::error(format!("Screenshot failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/scroll.rsnew file mode 100644index 0000000000..ea8b642da1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/scroll.rs@@ -0,0 +1,113 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct ScrollTool {+ state: Arc<ToolState>,+}++impl ScrollTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "scroll".into(),+ description: "Scroll the target pid's focused region by synthesized keystrokes.\n\n\+ Mapping: by='page' → PageDown/PageUp × amount; by='line' → DownArrow/UpArrow × amount. \+ Horizontal variants use Left/Right arrow keys.\n\n\+ Optional element_index + window_id pre-focuses the element before scrolling.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "direction"],+ "properties": {+ "pid": { "type": "integer" },+ "direction": {+ "type": "string",+ "enum": ["up", "down", "left", "right"],+ "description": "Scroll direction."+ },+ "by": {+ "type": "string",+ "enum": ["line", "page"],+ "description": "Scroll granularity. Default: line."+ },+ "amount": {+ "type": "integer",+ "minimum": 1,+ "maximum": 50,+ "description": "Number of keystroke repetitions. Default: 3."+ },+ "window_id": { "type": "integer" },+ "element_index": { "type": "integer" }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ idempotent: false,+ open_world: true,+ })+}++#[async_trait]+impl Tool for ScrollTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let direction = match args.get("direction").and_then(|v| v.as_str()) {+ Some(d) => d.to_owned(),+ None => return ToolResult::error("Missing required parameter: direction"),+ };+ let by = args.get("by").and_then(|v| v.as_str()).unwrap_or("line").to_owned();+ let amount = args.get("amount").and_then(|v| v.as_u64()).unwrap_or(3) as usize;+ let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);++ // Pre-focus element if requested.+ if let (Some(idx), Some(wid)) = (element_index, window_id) {+ if let Some(element_ptr) = self.state.element_cache.get_element_ptr(pid, wid, idx) {+ let _ = tokio::task::spawn_blocking(move || {+ crate::input::ax_actions::focus_element(element_ptr)+ }).await;+ tokio::time::sleep(std::time::Duration::from_millis(30)).await;+ }+ }++ let key = match (by.as_str(), direction.as_str()) {+ ("page", "down") | (_, "down") if by == "page" => "pagedown",+ ("page", "up") | (_, "up") if by == "page" => "pageup",+ ("line", "down") | (_, "down") => "down",+ ("line", "up") | (_, "up") => "up",+ (_, "left") => "left",+ (_, "right") => "right",+ _ => "down",+ };+ let key = key.to_owned();++ let result = tokio::task::spawn_blocking(move || {+ for _ in 0..amount {+ if let Err(e) = crate::input::keyboard::press_key(pid, &key, &[]) {+ return Err(e);+ }+ std::thread::sleep(std::time::Duration::from_millis(50));+ }+ Ok(())+ }).await;++ match result {+ Ok(Ok(())) => ToolResult::text(format!("Scrolled {direction} by {by} × {amount}.")),+ Ok(Err(e)) => ToolResult::error(format!("Scroll failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/set_config.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/set_config.rsnew file mode 100644index 0000000000..81eef9b992--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/set_config.rs@@ -0,0 +1,61 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct SetConfigTool {+ state: Arc<ToolState>,+}++impl SetConfigTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "set_config".into(),+ description: "Update cua-driver-rs configuration. Changes take effect immediately.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {+ "capture_mode": {+ "type": "string",+ "enum": ["som", "vision", "ax"],+ "description": "Default capture mode for get_window_state."+ },+ "max_image_dimension": {+ "type": "integer",+ "description": "Max dimension for screenshot resizing (0 = no limit)."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for SetConfigTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let mut cfg = self.state.config.write().unwrap();+ if let Some(mode) = args.get("capture_mode").and_then(|v| v.as_str()) {+ cfg.capture_mode = mode.to_owned();+ }+ if let Some(dim) = args.get("max_image_dimension").and_then(|v| v.as_u64()) {+ cfg.max_image_dimension = dim as u32;+ }+ ToolResult::text(format!(+ "Config updated: capture_mode={}, max_image_dimension={}",+ cfg.capture_mode, cfg.max_image_dimension+ ))+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/set_value.rsnew file mode 100644index 0000000000..85d4e17cd2--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/set_value.rs@@ -0,0 +1,346 @@+//! set_value tool — matches the Swift reference in SetValueTool.swift.+//!+//! Two modes, determined by the element's AXRole:+//!+//! * **AXPopUpButton**: Find the child option whose AXTitle or AXValue matches+//! `value` (case-insensitive) and AXPress it directly. The native macOS popup+//! menu is never opened, so focus is never stolen. Falls back to Safari+//! `osascript do JavaScript` for WebKit `<select>` elements that expose no AX+//! children when the popup is closed.+//!+//! * **Everything else**: Write `AXValue` directly (sliders, steppers, native+//! text fields that expose a settable AXValue).++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use crate::ax::bindings::{+ copy_children, copy_string_attr, perform_action, set_string_attr,+ kAXErrorSuccess, AXUIElementRef,+};+use core_foundation::base::CFRelease;++use super::ToolState;++pub struct SetValueTool {+ state: Arc<ToolState>,+}++impl SetValueTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "set_value".into(),+ description:+ "Set a value on a UI element. Two modes depending on element role:\n\+ \n\+ - **AXPopUpButton / select dropdown**: finds the child option whose \+ title or value matches `value` (case-insensitive) and AXPresses it \+ directly — the native macOS popup menu is never opened, so focus \+ is never stolen. Use this for HTML <select> elements in Safari or \+ any native NSPopUpButton.\n\+ \n\+ - **All other elements**: writes AXValue directly (sliders, steppers, \+ date pickers, native text fields that expose settable AXValue).\n\+ \n\+ For free-form text entry into web inputs, prefer `type_text_chars` \+ which synthesises key events — AXValue writes are ignored by WebKit."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "window_id", "element_index", "value"],+ "properties": {+ "pid": { "type": "integer" },+ "window_id": {+ "type": "integer",+ "description": "CGWindowID for the window whose get_window_state produced the element_index."+ },+ "element_index": { "type": "integer" },+ "value": {+ "type": "string",+ "description": "New value. AX will coerce to the element's native type."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: true,+ open_world: true,+ })+}++#[async_trait]+impl Tool for SetValueTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let window_id = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: window_id"),+ };+ let element_index = match args.get("element_index").and_then(|v| v.as_u64()) {+ Some(v) => v as usize,+ None => return ToolResult::error("Missing required parameter: element_index"),+ };+ let value = match args.get("value").and_then(|v| v.as_str()) {+ Some(v) => v.to_owned(),+ None => return ToolResult::error("Missing required parameter: value"),+ };++ let element_ptr = match self.state.element_cache.get_element_ptr(pid, window_id, element_index) {+ Some(p) => p,+ None => return ToolResult::error(format!(+ "Element index {element_index} not found. Call get_window_state first."+ )),+ };++ let result = tokio::task::spawn_blocking(move || {+ set_value_blocking(element_ptr, element_index, pid, &value)+ }).await;++ match result {+ Ok(Ok(msg)) => ToolResult::text(msg),+ Ok(Err(e)) => ToolResult::error(format!("set_value failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── Blocking implementation (runs on spawn_blocking thread) ─────────────────++fn set_value_blocking(+ element_ptr: usize,+ element_index: usize,+ pid: i32,+ value: &str,+) -> anyhow::Result<String> {+ let element = element_ptr as AXUIElementRef;++ let role = unsafe { copy_string_attr(element, "AXRole") }+ .unwrap_or_default();++ if role == "AXPopUpButton" {+ let element_title = unsafe { copy_string_attr(element, "AXTitle") }+ .unwrap_or_default();+ select_popup_option(element, element_index, pid, value, &element_title)+ } else {+ // Default path: write AXValue directly.+ let err = unsafe { set_string_attr(element, "AXValue", value) };+ if err == kAXErrorSuccess {+ Ok(format!("✅ Set AXValue on [{element_index}] {role}."))+ } else {+ anyhow::bail!("AXUIElementSetAttributeValue(AXValue) failed with error {err}")+ }+ }+}++// ── AXPopUpButton path ───────────────────────────────────────────────────────++fn select_popup_option(+ element: AXUIElementRef,+ element_index: usize,+ pid: i32,+ value: &str,+ element_title: &str,+) -> anyhow::Result<String> {+ let children = unsafe { copy_children(element) };++ if !children.is_empty() {+ // Strategy 1: AX children (native AppKit NSPopUpButton).+ let value_lower = value.to_lowercase();+ let mut matched_idx: Option<usize> = None;+ let mut available: Vec<String> = Vec::with_capacity(children.len());++ for (i, &child) in children.iter().enumerate() {+ let child_title = unsafe { copy_string_attr(child, "AXTitle") }+ .unwrap_or_default();+ let child_value = unsafe { copy_string_attr(child, "AXValue") }+ .unwrap_or_default();+ available.push(child_title.clone());+ if child_title.to_lowercase() == value_lower+ || child_value.to_lowercase() == value_lower+ {+ matched_idx = Some(i);+ break;+ }+ }++ let result = if let Some(i) = matched_idx {+ let child = children[i];+ let opt_title = unsafe { copy_string_attr(child, "AXTitle") }+ .unwrap_or_else(|| value.to_string());+ let err = unsafe { perform_action(child, "AXPress") };+ if err == kAXErrorSuccess {+ Ok(format!(+ "✅ Selected '{opt_title}' in AXPopUpButton [{element_index}] \+ \"{element_title}\" via AX child AXPress."+ ))+ } else {+ anyhow::bail!(+ "AXPress on child option failed with error {err}"+ )+ }+ } else {+ let avail = available.iter()+ .map(|t| format!("\"{t}\""))+ .collect::<Vec<_>>()+ .join(", ");+ anyhow::bail!(+ "No AX child matching '{value}' in AXPopUpButton [{element_index}] \+ \"{element_title}\". Available: [{avail}]"+ )+ };++ // Release children (copy_children retains each one).+ for &child in &children {+ unsafe { CFRelease(child as _); }+ }++ return result;+ }++ // Strategy 2: Safari/WebKit — no AX children when popup is closed.+ // Use osascript do JavaScript to set the <select> element's DOM value.+ let app_name = crate::apps::get_app_name_for_pid(pid)+ .unwrap_or_default();++ if app_name != "Safari" {+ anyhow::bail!(+ "AXPopUpButton [{element_index}] '{element_title}' has no AX children and \+ target is '{app_name}' (not Safari) — no fallback available."+ )+ }++ set_select_via_js(element_index, element_title, value)+}++// ── Safari JavaScript fallback ───────────────────────────────────────────────++/// Set an HTML `<select>` value in Safari via `osascript do JavaScript`.+/// Searches all `<select>` elements for an `<option>` whose text or value matches+/// `value` (case-insensitive), then sets it and dispatches a `change` event.+fn set_select_via_js(+ element_index: usize,+ element_title: &str,+ value: &str,+) -> anyhow::Result<String> {+ // Percent-encode the lowercased value using only unreserved URL characters+ // as the allowed set, matching the Swift reference's percent-encoding approach.+ // This makes the string safe to embed in both a JS single-quoted string+ // (via decodeURIComponent) and an AppleScript double-quoted string.+ let v_low = value.to_lowercase();+ let v_encoded = percent_encode_unreserved(&v_low);++ // JavaScript that matches the Swift reference verbatim.+ let js = format!(+ "(function(){{\+ var v=decodeURIComponent('{v_encoded}');\+ var ss=document.querySelectorAll('select'),opts=[];\+ for(var i=0;i<ss.length;i++){{\+ for(var j=0;j<ss[i].options.length;j++){{\+ var t=ss[i].options[j].text.toLowerCase(),\+ u=ss[i].options[j].value.toLowerCase();\+ opts.push(t+'|'+u);\+ if(t===v||u===v){{\+ ss[i].value=ss[i].options[j].value;\+ ss[i].dispatchEvent(new Event('change',{{bubbles:true}}));\+ return 'SET:'+ss[i].value;}}}}\+ }}return 'NOTFOUND:'+opts.join(',');\+ }})()"+ );++ let apple_script = format!(+ "tell application \"Safari\" to do JavaScript \"{js}\" in front document"+ );++ // Spawn osascript with a 10-second deadline. A stuck Safari permission+ // prompt or unresponsive renderer can cause wait() to block indefinitely,+ // which would stall the MCP tool handler permanently.+ let mut child = std::process::Command::new("osascript")+ .arg("-e")+ .arg(&apple_script)+ .stdout(std::process::Stdio::piped())+ .stderr(std::process::Stdio::piped())+ .spawn()+ .map_err(|e| anyhow::anyhow!("osascript launch failed: {e}"))?;++ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);+ loop {+ match child.try_wait() {+ Ok(Some(_)) => break,+ Ok(None) => {+ if std::time::Instant::now() >= deadline {+ let _ = child.kill();+ anyhow::bail!("osascript timed out after 10 seconds");+ }+ std::thread::sleep(std::time::Duration::from_millis(50));+ }+ Err(e) => anyhow::bail!("osascript wait error: {e}"),+ }+ }+ let out = child.wait_with_output()+ .map_err(|e| anyhow::anyhow!("osascript output error: {e}"))?;++ let raw = String::from_utf8_lossy(&out.stdout)+ .trim()+ .to_string();++ if raw.starts_with("SET:") {+ let dom_val = &raw[4..];+ Ok(format!(+ "✅ Set select [{element_index}] '{element_title}' to '{value}' via \+ Safari JavaScript (DOM value: \"{dom_val}\")."+ ))+ } else if raw.starts_with("NOTFOUND:") {+ let available = &raw[9..];+ anyhow::bail!(+ "No <option> matching '{value}' found in any <select>. \+ Available (text|value): {available}"+ )+ } else if raw.is_empty() && !out.status.success() {+ let err_text = String::from_utf8_lossy(&out.stderr);+ anyhow::bail!("osascript failed: {}", err_text.trim())+ } else {+ anyhow::bail!(+ "JavaScript returned unexpected output: {}",+ &raw[..raw.len().min(200)]+ )+ }+}++// ── Percent-encoding helper ──────────────────────────────────────────────────++/// Percent-encode a string, leaving only unreserved URL characters (`-._~` ++/// alphanumerics) unencoded. Matches the Swift reference's approach.+fn percent_encode_unreserved(s: &str) -> String {+ let mut out = String::with_capacity(s.len() * 3);+ for b in s.bytes() {+ if b.is_ascii_alphanumeric() || b == b'-' || b == b'.' || b == b'_' || b == b'~' {+ out.push(b as char);+ } else {+ out.push('%');+ out.push(hex_digit(b >> 4));+ out.push(hex_digit(b & 0xF));+ }+ }+ out+}++fn hex_digit(n: u8) -> char {+ match n {+ 0..=9 => (b'0' + n) as char,+ 10..=15 => (b'A' + n - 10) as char,+ _ => '0',+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text.rsnew file mode 100644index 0000000000..8040e0b4eb--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text.rs@@ -0,0 +1,231 @@+//! type_text tool — matches the Swift reference TypeTextTool.swift.+//!+//! Inserts text via `AXSelectedText` attribute write — an atomic single-call+//! insertion at the current cursor position. This is the preferred path for+//! all standard Cocoa text views (NSTextField, NSTextView, WKWebView text+//! inputs in Safari, etc.) and is significantly faster than per-keystroke+//! CGEvent synthesis.+//!+//! For Chromium / Electron inputs that don't implement `kAXSelectedText`,+//! the tool falls back to character-by-character CGEvent keystrokes so the+//! caller doesn't need to detect the app type themselves.+//!+//! Use `type_text_chars` when you explicitly need per-character pacing+//! (e.g., to trigger live-search debounce handlers).++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use crate::ax::bindings::{+ copy_string_attr, focused_element_of_pid, kAXErrorSuccess, set_string_attr,+ AXUIElementRef,+};+use core_foundation::base::CFRelease;++use super::ToolState;++pub struct TypeTextTool {+ pub state: Arc<ToolState>,+}++impl TypeTextTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "type_text".into(),+ description:+ "Insert text into the target pid via `AXSetAttribute(kAXSelectedText)`. \+ Works for standard Cocoa text fields and text views. No keystrokes are \+ synthesized — special keys (Return / Escape / arrows) go through \+ `press_key` / `hotkey`. For Chromium / Electron inputs that don't \+ implement `kAXSelectedText`, the tool falls back to CGEvent \+ character synthesis automatically.\n\n\+ Optional `element_index` + `window_id` (from the last \+ `get_window_state` snapshot) directs the write to a specific field. \+ Without `element_index`, the write goes to the pid's currently \+ focused element."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "text"],+ "properties": {+ "pid": { "type": "integer", "description": "Target process ID." },+ "text": { "type": "string", "description": "Text to insert at the target's cursor." },+ "window_id": {+ "type": "integer",+ "description": "CGWindowID. Required when element_index is used."+ },+ "element_index": {+ "type": "integer",+ "description": "Element index from last get_window_state. Directs the write to a specific field. Requires window_id."+ },+ "delay_ms": {+ "type": "integer",+ "minimum": 0,+ "maximum": 200,+ "description": "Milliseconds between characters in the CGEvent fallback path. Default 30. Ignored when the AX path succeeds."+ }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+}++#[async_trait]+impl Tool for TypeTextTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let text = match args.get("text").and_then(|v| v.as_str()) {+ Some(v) => v.to_owned(),+ None => return ToolResult::error("Missing required parameter: text"),+ };+ let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);+ let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30);++ // Validate element_index requires window_id.+ if element_index.is_some() && window_id.is_none() {+ return ToolResult::error(+ "window_id is required when element_index is used."+ );+ }++ // Resolve the element pointer (if element_index given).+ let element_ptr = if let (Some(idx), Some(wid)) = (element_index, window_id) {+ match self.state.element_cache.get_element_ptr(pid, wid, idx) {+ Some(p) => Some((p, Some(idx))),+ None => return ToolResult::error(format!(+ "Element index {idx} not found. Call get_window_state first."+ )),+ }+ } else {+ None+ };++ let text_clone = text.clone();+ let char_count = text.chars().count();++ let result = tokio::task::spawn_blocking(move || {+ type_text_blocking(pid, &text_clone, element_ptr, delay_ms)+ }).await;++ match result {+ Ok(Ok(detail)) => ToolResult::text(format!(+ "✅ Inserted {char_count} char(s){detail}."+ )),+ Ok(Err(e)) => ToolResult::error(format!("type_text failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── Blocking implementation ───────────────────────────────────────────────────++/// `element_ptr_and_idx` — `Some((ptr, idx))` if element_index was provided.+///+/// Returns a short detail string appended to the success message.+fn type_text_blocking(+ pid: i32,+ text: &str,+ element_ptr_and_idx: Option<(usize, Option<usize>)>,+ delay_ms: u64,+) -> anyhow::Result<String> {+ // --- Path 1: caller provided an explicit element index ---+ if let Some((ptr, idx_opt)) = element_ptr_and_idx {+ let element = ptr as AXUIElementRef;+ let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default();+ let title = unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default();++ let err = unsafe { set_string_attr(element, "AXSelectedText", text) };+ if err == kAXErrorSuccess {+ // Verify the write was actually applied — Chromium web inputs silently+ // accept the call but never update their DOM value. Read AXValue back;+ // if it's still empty when we just wrote non-empty text the write was a+ // no-op and we must use CGEvent instead.+ let silent_accept = !text.is_empty() && unsafe {+ copy_string_attr(element, "AXValue")+ .map(|v| v.is_empty())+ .unwrap_or(false)+ };+ if !silent_accept {+ let idx_str = idx_opt.map(|i| format!(" [{i}]")).unwrap_or_default();+ return Ok(format!(" into{idx_str} {role} \"{title}\""));+ }+ tracing::debug!(+ "AXSelectedText silent-accept detected for {role} \"{title}\" \+ (AXValue still empty), falling back to CGEvent keystrokes"+ );+ } else {+ // AXSelectedText not supported (Chromium/Electron) — fall through to CGEvent.+ tracing::debug!(+ "AXSelectedText write failed ({err}) for {role} \"{title}\", \+ falling back to CGEvent keystrokes"+ );+ }+ return crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)+ .map(|_| format!(" via CGEvent (AXSelectedText unsupported/silent-accept, {delay_ms}ms delay)"));+ }++ // --- Path 2: no element_index — target the pid's focused element ---+ let focused = unsafe { focused_element_of_pid(pid) };+ if let Some(element) = focused {+ let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default();+ let title = unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default();++ let err = unsafe { set_string_attr(element, "AXSelectedText", text) };++ let did_succeed = if err == kAXErrorSuccess {+ // Same silent-accept check: Chromium web inputs return success but+ // leave AXValue empty, meaning the write never reached the DOM.+ let silent_accept = !text.is_empty() && unsafe {+ copy_string_attr(element, "AXValue")+ .map(|v| v.is_empty())+ .unwrap_or(false)+ };+ if silent_accept {+ tracing::debug!(+ "AXSelectedText silent-accept detected for focused {role} \"{title}\" \+ (AXValue still empty), falling back to CGEvent keystrokes"+ );+ }+ !silent_accept+ } else {+ false+ };++ unsafe { CFRelease(element as _); }++ if did_succeed {+ return Ok(format!(" into focused {role} \"{title}\""));+ }+ if err != kAXErrorSuccess {+ // Fall through to CGEvent.+ tracing::debug!(+ "AXSelectedText write failed ({err}) for focused {role}, \+ falling back to CGEvent keystrokes"+ );+ }+ } else {+ tracing::debug!(+ "No focused element found for pid {pid}, falling back to CGEvent keystrokes"+ );+ }++ crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)+ .map(|_| format!(" via CGEvent (no focused element / AXSelectedText unsupported, {delay_ms}ms delay)"))+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text_chars.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text_chars.rsnew file mode 100644index 0000000000..fb2628c8c5--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text_chars.rs@@ -0,0 +1,85 @@+use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::ToolState;++pub struct TypeTextCharsTool {+ state: Arc<ToolState>,+}++impl TypeTextCharsTool {+ pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "type_text_chars".into(),+ description: "Type text character-by-character with a configurable inter-character delay. \+ Useful for apps that miss keystrokes when characters arrive too quickly. \+ Otherwise identical to type_text (CGEvent, no focus steal).".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["pid", "text"],+ "properties": {+ "pid": { "type": "integer", "description": "Target process ID." },+ "text": { "type": "string", "description": "Text to type." },+ "delay_ms": { "type": "integer", "description": "Milliseconds between characters (default 30)." },+ "window_id": { "type": "integer", "description": "Window ID for element focus." },+ "element_index": { "type": "integer", "description": "Element to focus before typing." },+ "type_chars_only": { "type": "boolean", "description": "Skip AX focus, type directly. Default false." }+ },+ "additionalProperties": false+ }),+ read_only: false,+ destructive: true,+ idempotent: false,+ open_world: true,+ })+}++#[async_trait]+impl Tool for TypeTextCharsTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as i32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let text = match args.get("text").and_then(|v| v.as_str()) {+ Some(v) => v.to_owned(),+ None => return ToolResult::error("Missing required parameter: text"),+ };+ let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30);+ let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);+ let type_chars_only = args.get("type_chars_only").and_then(|v| v.as_bool()).unwrap_or(false);++ // Pre-focus element if requested.+ if !type_chars_only {+ if let (Some(idx), Some(wid)) = (element_index, window_id) {+ if let Some(element_ptr) = self.state.element_cache.get_element_ptr(pid, wid, idx) {+ let _ = tokio::task::spawn_blocking(move || {+ crate::input::ax_actions::focus_element(element_ptr)+ }).await;+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;+ }+ }+ }++ let text_len = text.chars().count();+ let result = tokio::task::spawn_blocking(move || {+ crate::input::keyboard::type_text_with_delay(pid, &text, delay_ms)+ }).await;++ match result {+ Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) with {delay_ms}ms delay.")),+ Ok(Err(e)) => ToolResult::error(format!("Type text chars failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/zoom.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/zoom.rsnew file mode 100644index 0000000000..0ff2ab6c6b--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/zoom.rs@@ -0,0 +1,109 @@+use async_trait::async_trait;+use base64::{engine::general_purpose::STANDARD as BASE64, Engine};+use mcp_server::{protocol::{ToolResult, Content}, tool::{Tool, ToolDef}};+use serde_json::Value;+use std::sync::Arc;++use super::{ToolState, ZoomContext};++pub struct ZoomTool {+ pub state: Arc<ToolState>,+}++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "zoom".into(),+ description: "Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in screenshot \+ pixel coordinates, with 20% padding added on each side. The output image is at most \+ 500 px wide.\n\n\+ After a zoom, pass `from_zoom=true` to click/type_text to auto-translate coordinates \+ back to full-window space.".into(),+ input_schema: serde_json::json!({+ "type": "object",+ "required": ["window_id", "x1", "y1", "x2", "y2"],+ "properties": {+ "window_id": { "type": "integer", "description": "CGWindowID from list_windows." },+ "pid": { "type": "integer", "description": "Target pid — required for from_zoom click/type translation." },+ "x1": { "type": "number", "description": "Left edge of region in screenshot pixels." },+ "y1": { "type": "number", "description": "Top edge of region in screenshot pixels." },+ "x2": { "type": "number", "description": "Right edge of region in screenshot pixels." },+ "y2": { "type": "number", "description": "Bottom edge of region in screenshot pixels." }+ },+ "additionalProperties": false+ }),+ read_only: true,+ destructive: false,+ idempotent: true,+ open_world: false,+ })+}++#[async_trait]+impl Tool for ZoomTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ let window_id = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: window_id"),+ };+ let pid = args.get("pid").and_then(|v| v.as_i64()).map(|v| v as i32);+ let x1 = match args.get("x1").and_then(|v| v.as_f64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: x1"),+ };+ let y1 = match args.get("y1").and_then(|v| v.as_f64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: y1"),+ };+ let x2 = match args.get("x2").and_then(|v| v.as_f64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: x2"),+ };+ let y2 = match args.get("y2").and_then(|v| v.as_f64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required parameter: y2"),+ };++ if x2 <= x1 || y2 <= y1 {+ return ToolResult::error("x2 must be > x1 and y2 must be > y1");+ }++ let state = self.state.clone();+ let result = tokio::task::spawn_blocking(move || {+ let png_bytes = crate::capture::screenshot_window_bytes(window_id)?;+ cursor_overlay::capture_utils::crop_png_to_jpeg(&png_bytes, x1, y1, x2, y2, 500)+ }).await;++ match result {+ Ok(Ok(crop)) => {+ // Store zoom context so from_zoom clicks can translate back.+ if let Some(p) = pid {+ state.zoom_registry.set(p, ZoomContext {+ origin_x: crop.origin_x,+ origin_y: crop.origin_y,+ scale_inv: crop.scale_inv,+ });+ }+ let (w, h) = (crop.out_w, crop.out_h);+ let b64 = BASE64.encode(&crop.jpeg_bytes);+ ToolResult {+ content: vec![+ Content::image_jpeg(b64),+ Content::text(format!(+ "Zoom region ({x1:.0},{y1:.0})–({x2:.0},{y2:.0}) → {w}×{h} px JPEG."+ )),+ ],+ is_error: None,+ structured_content: Some(serde_json::json!({+ "width": w, "height": h, "format": "jpeg"+ })),+ }+ }+ Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/windows.rs b/libs/cua-driver-rs/crates/platform-macos/src/windows.rsnew file mode 100644index 0000000000..3aa6392d0e--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-macos/src/windows.rs@@ -0,0 +1,232 @@+//! macOS window enumeration via CGWindowList APIs.+//!+//! Uses the C-level CGWindowListCopyWindowInfo API which returns a CFArray+//! of CFDictionary objects describing each window.++use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct WindowBounds {+ pub x: f64,+ pub y: f64,+ pub width: f64,+ pub height: f64,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct WindowInfo {+ pub window_id: u32,+ pub pid: i32,+ pub app_name: String,+ pub title: String,+ pub bounds: WindowBounds,+ pub layer: i32,+ pub z_index: usize,+ pub is_on_screen: bool,+ pub on_current_space: Option<bool>,+ pub space_ids: Option<Vec<u64>>,+}++// ── CGWindow option flags ─────────────────────────────────────────────────────+const kCGWindowListExcludeDesktopElements: u32 = 16;+const kCGWindowListOptionOnScreenOnly: u32 = 1;+const kCGNullWindowID: u32 = 0;++// ── Internal CGWindowInfo parsing ─────────────────────────────────────────────+//+// We use `system_profiler` workaround via `CGWindowListCopyWindowInfo` which+// returns a plist-like structure. The simplest cross-compile-safe approach+// is to dump via `osascript` or use the Objective-C runtime.+//+// For the initial version we use the `core-foundation` crate + direct C linkage.++#[link(name = "CoreGraphics", kind = "framework")]+extern "C" {+ fn CGWindowListCopyWindowInfo(+ option: u32,+ relativeToWindow: u32,+ ) -> core_foundation::array::CFArrayRef;+}++/// Enumerate all windows (including off-screen).+pub fn all_windows() -> Vec<WindowInfo> {+ enumerate_windows(kCGWindowListExcludeDesktopElements)+}++/// Enumerate only on-screen windows.+pub fn visible_windows() -> Vec<WindowInfo> {+ enumerate_windows(kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements)+}++fn enumerate_windows(options: u32) -> Vec<WindowInfo> {+ use core_foundation::{+ array::CFArray,+ base::{CFGetTypeID, TCFType, CFTypeRef},+ dictionary::CFDictionary,+ string::CFString,+ number::CFNumber,+ boolean::CFBoolean,+ };+ use std::os::raw::c_void;++ let raw_ref = unsafe {+ CGWindowListCopyWindowInfo(options, kCGNullWindowID)+ };+ if raw_ref.is_null() {+ return vec![];+ }++ let raw: CFArray<CFTypeRef> = unsafe { CFArray::wrap_under_create_rule(raw_ref as _) };+ let total = raw.len() as usize;+ let mut result = Vec::new();++ for (idx, item) in raw.iter().enumerate() {+ let item = *item;+ // Each item should be a CFDictionary.+ let dict_type = CFDictionary::<*const c_void, *const c_void>::type_id();+ if unsafe { CFGetTypeID(item) } != dict_type {+ continue;+ }++ let dict: CFDictionary<*const c_void, *const c_void> = unsafe {+ CFDictionary::wrap_under_get_rule(item as _)+ };++ // Helper: get number from dict by key string.+ let get_num = |key: &str| -> i64 {+ let k = CFString::new(key);+ dict.find(k.as_concrete_TypeRef() as *const c_void)+ .and_then(|v| unsafe {+ let v = *v;+ if CFGetTypeID(v) == CFNumber::type_id() {+ CFNumber::wrap_under_get_rule(v as _).to_i64()+ } else { None }+ })+ .unwrap_or(0)+ };++ let get_str = |key: &str| -> String {+ let k = CFString::new(key);+ dict.find(k.as_concrete_TypeRef() as *const c_void)+ .and_then(|v| unsafe {+ let v = *v;+ if CFGetTypeID(v) == CFString::type_id() {+ Some(CFString::wrap_under_get_rule(v as _).to_string())+ } else { None }+ })+ .unwrap_or_default()+ };++ let get_bool = |key: &str| -> bool {+ let k = CFString::new(key);+ dict.find(k.as_concrete_TypeRef() as *const c_void)+ .map(|v| unsafe {+ let v = *v;+ if CFGetTypeID(v) == CFBoolean::type_id() {+ bool::from(CFBoolean::wrap_under_get_rule(v as _))+ } else { false }+ })+ .unwrap_or(false)+ };++ let window_id = get_num("kCGWindowNumber") as u32;+ let pid = get_num("kCGWindowOwnerPID") as i32;+ let app_name = get_str("kCGWindowOwnerName");+ let title = get_str("kCGWindowName");+ let layer = get_num("kCGWindowLayer") as i32;+ let is_on_screen = get_bool("kCGWindowIsOnscreen");++ // Only include layer-0 windows.+ if layer != 0 { continue; }++ // Parse bounds dict.+ let bounds = {+ let bk = CFString::new("kCGWindowBounds");+ dict.find(bk.as_concrete_TypeRef() as *const c_void)+ .and_then(|v| unsafe {+ let v = *v;+ if CFGetTypeID(v) == CFDictionary::<*const c_void, *const c_void>::type_id() {+ let bd: CFDictionary<*const c_void, *const c_void> =+ CFDictionary::wrap_under_get_rule(v as _);+ let x = get_bounds_num(&bd, "X");+ let y = get_bounds_num(&bd, "Y");+ let w = get_bounds_num(&bd, "Width");+ let h = get_bounds_num(&bd, "Height");+ Some(WindowBounds { x, y, width: w, height: h })+ } else { None }+ })+ .unwrap_or(WindowBounds { x: 0., y: 0., width: 0., height: 0. })+ };++ // z_index: CGWindowList front-to-back → assign reverse index.+ let z_index = total - idx;++ result.push(WindowInfo {+ window_id,+ pid,+ app_name,+ title,+ bounds,+ layer,+ z_index,+ is_on_screen,+ on_current_space: None,+ space_ids: None,+ });+ }++ result+}++fn get_bounds_num(+ dict: &core_foundation::dictionary::CFDictionary<*const std::os::raw::c_void, *const std::os::raw::c_void>,+ key: &str,+) -> f64 {+ use core_foundation::{+ base::{CFGetTypeID, TCFType},+ number::CFNumber,+ string::CFString,+ };+ use std::os::raw::c_void;++ let k = CFString::new(key);+ dict.find(k.as_concrete_TypeRef() as *const c_void)+ .and_then(|v| unsafe {+ let v = *v;+ if CFGetTypeID(v) == CFNumber::type_id() {+ CFNumber::wrap_under_get_rule(v as _).to_f64()+ } else { None }+ })+ .unwrap_or(0.0)+}++/// Look up a window's bounds by its CGWindowID.+///+/// Returns `None` if the window is not currently known to WindowServer+/// (e.g. it was closed or the window_id is stale).+pub fn window_bounds_by_id(window_id: u32) -> Option<WindowBounds> {+ all_windows()+ .into_iter()+ .find(|w| w.window_id == window_id)+ .map(|w| w.bounds)+}++/// Select the best window_id for a pid.+pub fn resolve_main_window_id(pid: i32) -> anyhow::Result<u32> {+ let windows = all_windows();+ let pid_windows: Vec<&WindowInfo> = windows.iter().filter(|w| w.pid == pid).collect();+ if pid_windows.is_empty() {+ anyhow::bail!("pid {pid} has no windows");+ }+ let mut on_screen: Vec<&&WindowInfo> = pid_windows.iter().filter(|w| w.is_on_screen).collect();+ if !on_screen.is_empty() {+ on_screen.sort_by(|a, b| b.z_index.cmp(&a.z_index));+ return Ok(on_screen[0].window_id);+ }+ let largest = pid_windows.iter().max_by(|a, b| {+ let area_a = a.bounds.width * a.bounds.height;+ let area_b = b.bounds.width * b.bounds.height;+ area_a.partial_cmp(&area_b).unwrap_or(std::cmp::Ordering::Equal)+ });+ Ok(largest.unwrap().window_id)+}diff --git a/libs/cua-driver-rs/crates/platform-windows/Cargo.toml b/libs/cua-driver-rs/crates/platform-windows/Cargo.tomlnew file mode 100644index 0000000000..0d892b396d--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/Cargo.toml@@ -0,0 +1,40 @@+[package]+name = "platform-windows"+version.workspace = true+edition.workspace = true++[dependencies]+tokio = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+anyhow = { workspace = true }+thiserror = { workspace = true }+tracing = { workspace = true }+async-trait = "0.1"+mcp-server = { path = "../mcp-server" }+cursor-overlay = { path = "../cursor-overlay" }+# tiny-skia for cross-platform cursor rendering (used in overlay.rs on all targets)+tiny-skia = { version = "0.11", default-features = false, features = ["std"] }++[target.'cfg(target_os = "windows")'.dependencies]+image = { workspace = true }+windows = { version = "0.58", features = [+ "Win32_Foundation",+ "Win32_UI_Accessibility",+ "Win32_UI_WindowsAndMessaging",+ "Win32_System_Threading",+ "Win32_System_Diagnostics_ToolHelp",+ "Win32_Graphics_Gdi",+ "Win32_Graphics_Dwm",+ "Win32_UI_Input_KeyboardAndMouse",+ "Win32_UI_HiDpi",+ "Win32_System_LibraryLoader",+ "Win32_Graphics_OpenGL",+ "Win32_Security",+ "Win32_System_Com",+ "Win32_Storage_Xps",+ "Win32_UI_Shell",+ "Win32_Media",+ "Win32_System_Registry",+] }+base64 = { workspace = true }diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/agent_cursor_setters_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/agent_cursor_setters_parity.rsnew file mode 100644index 0000000000..719520cca4--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/agent_cursor_setters_parity.rs@@ -0,0 +1,70 @@+//! Parity check for `set_agent_cursor_enabled` + `set_agent_cursor_motion`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. set_agent_cursor_enabled missing field error.+ let r1 = req(&mut pipe, r#"{"method":"call","name":"set_agent_cursor_enabled","args":{}}"#);+ let e1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert!(e1.to_lowercase().contains("enabled") && e1.to_lowercase().contains("required"),+ "Missing-enabled wording wrong: {e1:?}");+ println!("Missing-enabled err OK");++ // 2. set_agent_cursor_enabled true.+ let r2 = req(&mut pipe, r#"{"method":"call","name":"set_agent_cursor_enabled","args":{"enabled":true}}"#);+ let t2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ assert_eq!(t2, "✅ Agent cursor enabled.", "Enabled text wrong: {t2:?}");+ println!("Enabled OK");++ // 3. set_agent_cursor_enabled false.+ let r3 = req(&mut pipe, r#"{"method":"call","name":"set_agent_cursor_enabled","args":{"enabled":false}}"#);+ let t3 = extract_text(&serde_json::from_str(r3.trim()).unwrap());+ assert_eq!(t3, "✅ Agent cursor disabled.", "Disabled text wrong: {t3:?}");+ println!("Disabled OK");++ // Re-enable for subsequent tests.+ let _ = req(&mut pipe, r#"{"method":"call","name":"set_agent_cursor_enabled","args":{"enabled":true}}"#);++ // 4. set_agent_cursor_motion — tune knobs.+ let r4 = req(&mut pipe,+ r#"{"method":"call","name":"set_agent_cursor_motion","args":{"start_handle":0.4,"arc_size":0.3,"spring":0.8,"glide_duration_ms":500}}"#);+ let v4: serde_json::Value = serde_json::from_str(r4.trim()).unwrap();+ let t4 = extract_text(&v4);+ println!("Motion resp: {t4:?}");+ assert!(t4.starts_with("✅ cursor motion: startHandle=0.4 endHandle=") && t4.contains("arcSize=0.3")+ && t4.contains("spring=0.8") && t4.contains("glideDurationMs=500"),+ "Motion text doesn't match Swift format: {t4:?}");++ println!("\n✅ PASS: set_agent_cursor_enabled + set_agent_cursor_motion match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/click_domains.rs b/libs/cua-driver-rs/crates/platform-windows/examples/click_domains.rsnew file mode 100644index 0000000000..db131bb8e1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/click_domains.rs@@ -0,0 +1,75 @@+//! Click the "Domains" hyperlink in Chrome via the daemon pipe.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let chrome_pid = 62156u64;+ let chrome_wid = 4464038u64;++ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(8);+ loop {+ if Instant::now() > deadline { break; }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ // Enable cursor.+ req(&mut pipe, r#"{"method":"call","name":"set_agent_cursor_enabled","args":{"enabled":true}}"#);++ // Get window state with "Domains" query filter.+ let state_req = format!(+ r#"{{"method":"call","name":"get_window_state","args":{{"pid":{},"window_id":{},"query":"Domains"}}}}"#,+ chrome_pid, chrome_wid);+ let state_resp = req(&mut pipe, &state_req);+ // Find the Domains element index in the tree text. Tree format:+ // "- [N] Hyperlink \"Domains\" ..."+ // Find any clickable element whose name contains "Domains" — search the+ // \"Domains\" or "Domain" name.+ let query = std::env::args().nth(1).unwrap_or_else(|| "Domains".to_owned());+ let needle = format!("\"{}\"", query);+ let candidates: Vec<(u64, String)> = state_resp+ .split("\\n")+ .filter(|l| l.contains(&needle) && l.contains("- ["))+ .filter_map(|l| {+ let after_bracket = l.split("- [").nth(1)?;+ let n_str = after_bracket.split(']').next()?;+ let idx: u64 = n_str.parse().ok()?;+ Some((idx, l.to_owned()))+ }).collect();+ if candidates.is_empty() {+ eprintln!("No clickable element matching {needle}.");+ eprintln!("Tree dump (first 2000 chars):");+ eprintln!("{}", &state_resp.replace("\\n", "\n").chars().take(2000).collect::<String>());+ std::process::exit(1);+ }+ for (i, (idx, line)) in candidates.iter().enumerate() {+ println!(" candidate #{i}: [{idx}] {}", line.trim());+ }+ let (idx, _) = candidates[0];+ println!("Using element_index={idx}");++ let click_req = format!(+ r#"{{"method":"call","name":"click","args":{{"pid":{},"window_id":{},"element_index":{}}}}}"#,+ chrome_pid, chrome_wid, idx);+ let click_resp = req(&mut pipe, &click_req);+ println!("Click resp: {}", click_resp.trim());+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/click_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/click_parity.rsnew file mode 100644index 0000000000..9e8474bf53--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/click_parity.rs@@ -0,0 +1,119 @@+//! Parity check for `click` tool text format.+//!+//! Drives the daemon and asserts:+//! - Missing target returns Swift's wording.+//! - Pixel click against a Chrome window returns+//! `"✅ Posted click to pid X."`.+//! - Element-indexed click returns `"✅ Performed Invoke on [N] ..."`.+//!+//! Hard 4-second timeout per call.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe — start daemon first");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("response timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ // Helper: extract the user-visible text from either daemon error+ // wrapper or the MCP isError tool-result shape.+ fn extract_text(v: &serde_json::Value) -> String {+ if let Some(t) = v.pointer("/result/content/0/text").and_then(|t| t.as_str()) {+ return t.to_owned();+ }+ if let Some(e) = v.pointer("/error").and_then(|e| e.as_str()) {+ return e.to_owned();+ }+ format!("(no text in response: {v})")+ }++ // 1. Missing-target error wording. Need a real pid + window_id so the+ // early "no window" guard passes; we'll fetch a real one below first.+ // (moved after list_windows)++ // 2. Pixel click — pick the first window we can find belonging to our own+ // daemon (cua-driver.exe doesn't have one, so use the foreground app).+ // Use list_windows to get a real pid + hwnd, then click on it.+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).expect("parse");+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array())+ .expect("no windows array");+ // Prefer Chrome — we already verified it accepts PostMessage clicks+ // earlier in the session. Skip windows that reject PostMessage+ // (e.g. parsecd, system processes).+ let preferred = ["chrome.exe", "msedge.exe", "firefox.exe", "notepad.exe"];+ let target = wins.iter().find(|w| {+ let name = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ preferred.iter().any(|p| name.contains(p))+ }).or_else(|| wins.iter().find(|w| {+ let name = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("");+ !name.contains("cua-driver") && !name.contains("parsecd")+ && !name.contains("examples") && !name.contains("dwm")+ })).expect("no suitable target window");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();+ println!("Clicking pid={pid} window_id={wid} app={}", target.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("?"));++ // 1. Missing-target wording (now that we have a real pid + window_id).+ let missing_req = format!(+ r#"{{"method":"call","name":"click","args":{{"pid":{pid},"window_id":{wid}}}}}"#+ );+ let r_m = req(&mut pipe, &missing_req);+ let v_m: serde_json::Value = serde_json::from_str(r_m.trim()).expect("parse");+ let err1 = extract_text(&v_m);+ assert!(+ err1.contains("Provide element_index or (x, y) to address the click target"),+ "Expected Swift-style missing-target wording, got: {err1:?}"+ );+ println!("Missing-target err OK: {err1:?}");++ // Pixel click at (100, 100) window-local — innocuous enough.+ let click_req = format!(+ r#"{{"method":"call","name":"click","args":{{"pid":{pid},"window_id":{wid},"x":100,"y":100}}}}"#+ );+ let r2 = req(&mut pipe, &click_req);+ let v2: serde_json::Value = serde_json::from_str(r2.trim()).expect("parse");+ let text2 = extract_text(&v2);+ println!("Pixel-click resp: {text2:?}");+ assert!(+ text2.starts_with("✅ Posted click to pid "),+ "Expected `✅ Posted click to pid X.` Swift format, got: {text2:?}"+ );++ // 3. Double-click should produce `Posted double-click`.+ let click2 = format!(+ r#"{{"method":"call","name":"click","args":{{"pid":{pid},"window_id":{wid},"x":100,"y":100,"count":2}}}}"#+ );+ let r3 = req(&mut pipe, &click2);+ let v3: serde_json::Value = serde_json::from_str(r3.trim()).expect("parse");+ let text3 = extract_text(&v3);+ println!("Double-click resp: {text3:?}");+ assert!(+ text3.starts_with("✅ Posted double-click to pid "),+ "Expected `✅ Posted double-click to pid X.`, got: {text3:?}"+ );++ println!("\n✅ PASS: click error wording + click/double-click text format match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/config_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/config_parity.rsnew file mode 100644index 0000000000..80784c64e0--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/config_parity.rs@@ -0,0 +1,89 @@+//! Parity check for `get_config` + `set_config`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. get_config — text starts with "✅ " followed by pretty JSON.+ let r1 = req(&mut pipe, r#"{"method":"call","name":"get_config","args":{}}"#);+ let v1: serde_json::Value = serde_json::from_str(r1.trim()).unwrap();+ let t1 = extract_text(&v1);+ assert!(t1.starts_with("✅ "), "get_config text should start with '✅ ': {t1:?}");+ // Pretty JSON should contain schema_version and capture_mode keys.+ assert!(t1.contains("\"schema_version\"") && t1.contains("\"capture_mode\""),+ "get_config JSON missing keys: {t1:?}");+ // structuredContent should have the same fields.+ let sc1 = v1.pointer("/result/structuredContent").unwrap();+ assert!(sc1.get("schema_version").is_some());+ assert!(sc1.get("capture_mode").is_some());+ assert!(sc1.get("max_image_dimension").is_some());+ println!("get_config OK");++ // 2. set_config Swift-style {key, value}.+ let r2 = req(&mut pipe,+ r#"{"method":"call","name":"set_config","args":{"key":"capture_mode","value":"vision"}}"#);+ let v2: serde_json::Value = serde_json::from_str(r2.trim()).unwrap();+ let cm = v2.pointer("/result/structuredContent/capture_mode").and_then(|s| s.as_str());+ assert_eq!(cm, Some("vision"), "set_config didn't update capture_mode via {{key, value}}");+ println!("set_config Swift-style OK (capture_mode=vision)");++ // 3. set_config legacy per-field shape.+ let r3 = req(&mut pipe,+ r#"{"method":"call","name":"set_config","args":{"capture_mode":"som","max_image_dimension":1024}}"#);+ let v3: serde_json::Value = serde_json::from_str(r3.trim()).unwrap();+ let cm3 = v3.pointer("/result/structuredContent/capture_mode").and_then(|s| s.as_str());+ let md3 = v3.pointer("/result/structuredContent/max_image_dimension").and_then(|n| n.as_u64());+ assert_eq!(cm3, Some("som"));+ assert_eq!(md3, Some(1024));+ println!("set_config legacy OK (capture_mode=som, max=1024)");++ // 4. set_config unknown key error.+ let r4 = req(&mut pipe,+ r#"{"method":"call","name":"set_config","args":{"key":"bogus","value":42}}"#);+ let e4 = extract_text(&serde_json::from_str(r4.trim()).unwrap());+ assert!(e4.contains("Unknown config key"), "Unknown-key wording wrong: {e4:?}");+ println!("set_config unknown-key err OK");++ // 5. set_config missing inputs error.+ let r5 = req(&mut pipe, r#"{"method":"call","name":"set_config","args":{}}"#);+ let e5 = extract_text(&serde_json::from_str(r5.trim()).unwrap());+ assert!(e5.to_lowercase().contains("required") || e5.to_lowercase().contains("missing"),+ "Missing-input wording should mention required/missing: {e5:?}");+ println!("set_config missing-input err OK");++ // Restore defaults.+ let _ = req(&mut pipe,+ r#"{"method":"call","name":"set_config","args":{"capture_mode":"som","max_image_dimension":0}}"#);++ println!("\n✅ PASS: get_config + set_config (Swift-style and legacy) verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/cursor_visibility.rs b/libs/cua-driver-rs/crates/platform-windows/examples/cursor_visibility.rsnew file mode 100644index 0000000000..359197f19f--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/cursor_visibility.rs@@ -0,0 +1,131 @@+//! TDD harness for the agent cursor overlay — drives the LIVE DAEMON.+//!+//! Strategy: set the cursor to pure-magenta gradient (a colour that does not+//! naturally appear on screen), move it to a target, screenshot, and search+//! the screenshot for magenta pixels. Cleaner than diffing two screenshots+//! because we don't compete with screen activity.+//!+//! Hard 2-second timeout on every pipe round-trip.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ use image::GenericImageView;++ let target_x = 700i32;+ let target_y = 500i32;++ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe — start daemon first");+ println!("[1] connected");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 8192];+ let deadline = Instant::now() + Duration::from_secs(2);+ loop {+ if Instant::now() > deadline { break; }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ let r2 = req(&mut pipe, r#"{"method":"call","name":"set_agent_cursor_enabled","args":{"enabled":true}}"#);+ println!(" enabled resp: {}", r2.trim());+ // Set bright pure-magenta gradient — easy to find, won't match any normal screen content.+ let r3 = req(&mut pipe, r##"{"method":"call","name":"set_agent_cursor_style","args":{"gradient_colors":["#FF00FF","#FF00FF","#FF00FF"],"bloom_color":"#FF00FF"}}"##);+ println!(" style resp: {}", r3.trim());++ let cmd = format!(+ r#"{{"method":"call","name":"move_cursor","args":{{"x":{},"y":{}}}}}"#,+ target_x, target_y);+ let r4 = req(&mut pipe, &cmd);+ println!(" move resp: {}", r4.trim());++ // Poll until cursor centroid stops moving, with a 4s ceiling. Swift+ // signals arrival via a one-shot channel; the Rust Windows overlay doesn't+ // expose one yet, so we poll instead.+ let deadline = Instant::now() + Duration::from_secs(4);+ let mut last_centroid = (-1i64, -1i64);+ let mut stable_ticks = 0;+ let mut png;+ loop {+ std::thread::sleep(Duration::from_millis(150));+ png = platform_windows::capture::screenshot_display_bytes()+ .expect("screenshot");+ let img = image::load_from_memory(&png).expect("decode");+ let rgba = img.to_rgba8();+ let (iw, ih) = img.dimensions();+ let mut count = 0u64; let mut sx = 0i64; let mut sy = 0i64;+ for py in 0..ih { for px in 0..iw {+ let p = rgba.get_pixel(px, py).0;+ if p[0] > 180 && p[1] < 100 && p[2] > 180 {+ count += 1; sx += px as i64; sy += py as i64;+ }+ }}+ if count > 50 {+ let c = (sx / count as i64, sy / count as i64);+ let dist = ((c.0 - last_centroid.0).pow(2) + (c.1 - last_centroid.1).pow(2)) as f64;+ if dist.sqrt() < 3.0 { stable_ticks += 1; } else { stable_ticks = 0; }+ println!(" polling: centroid=({},{}) count={} stable={}", c.0, c.1, count, stable_ticks);+ last_centroid = c;+ if stable_ticks >= 3 { break; }+ }+ if Instant::now() > deadline { println!(" polling: deadline reached"); break; }+ }+ let _ = std::fs::write("O:/tmp_cursor_screen.png", &png);+ let img = image::load_from_memory(&png).expect("decode");+ let (w, h) = img.dimensions();+ let rgba = img.to_rgba8();++ // Search whole screen for magenta-ish pixels (R high, G low, B high).+ let mut magenta_count = 0u64;+ let mut sx = 0i64; let mut sy = 0i64;+ let mut min_x = w as i32; let mut max_x = 0i32;+ let mut min_y = h as i32; let mut max_y = 0i32;+ for py in 0..h {+ for px in 0..w {+ let p = rgba.get_pixel(px, py).0;+ // Tolerate antialiasing: R > 180, G < 100, B > 180+ if p[0] > 180 && p[1] < 100 && p[2] > 180 {+ magenta_count += 1;+ sx += px as i64; sy += py as i64;+ min_x = min_x.min(px as i32); max_x = max_x.max(px as i32);+ min_y = min_y.min(py as i32); max_y = max_y.max(py as i32);+ }+ }+ }+ println!("[4] screen {w}x{h}, magenta pixels: {magenta_count}");++ if magenta_count == 0 {+ eprintln!("\n❌ FAIL: NO magenta cursor pixels found anywhere on screen.");+ eprintln!(" The overlay window exists but isn't rendering, OR style cmd didn't apply.");+ std::process::exit(1);+ }++ let cx = sx / magenta_count as i64;+ let cy = sy / magenta_count as i64;+ let dx = (cx - target_x as i64).abs();+ let dy = (cy - target_y as i64).abs();+ println!("[5] centroid: ({cx},{cy}); bbox ({min_x},{min_y})-({max_x},{max_y})");+ println!(" target was ({target_x},{target_y}); offset = ({dx},{dy})");++ if dx > 100 || dy > 100 {+ eprintln!("\n❌ FAIL: cursor is rendering but at wrong location.");+ eprintln!(" Expected near ({target_x},{target_y}), found centroid ({cx},{cy}).");+ std::process::exit(2);+ }+ println!("\n✅ PASS: magenta cursor at ({cx},{cy}) close to target ({target_x},{target_y})");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/daemon_lifecycle_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/daemon_lifecycle_parity.rsnew file mode 100644index 0000000000..634d447894--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/daemon_lifecycle_parity.rs@@ -0,0 +1,58 @@+//! Parity check for `cua-driver status` + `cua-driver stop` CLI subcommands.++use std::process::Command;++#[cfg(target_os = "windows")]+fn main() {+ let exe = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .parent().unwrap()+ .parent().unwrap()+ .join("target/debug/cua-driver.exe");++ // Assume a daemon is running for these tests.++ // 1. status — should match Swift's 3-line output format.+ let out = Command::new(&exe).arg("status").output().expect("status");+ assert!(out.status.success(), "status should succeed when daemon is running");+ let stdout = String::from_utf8_lossy(&out.stdout);+ let lines: Vec<&str> = stdout.lines().collect();+ assert!(lines.len() >= 2, "status should print at least 2 lines, got: {stdout:?}");+ assert_eq!(lines[0], "cua-driver daemon is running",+ "first line wrong: {:?}", lines[0]);+ assert!(lines[1].trim_start().starts_with("socket:"),+ "second line should start with 'socket:': {:?}", lines[1]);+ if lines.len() >= 3 {+ assert!(lines[2].trim_start().starts_with("pid:"),+ "third line should start with 'pid:': {:?}", lines[2]);+ }+ println!("status OK");++ // 2. stop — should produce silent stdout on success (Swift parity).+ let out2 = Command::new(&exe).arg("stop").output().expect("stop");+ assert!(out2.status.success(), "stop should succeed when daemon is running");+ let stdout2 = String::from_utf8_lossy(&out2.stdout);+ assert!(stdout2.trim().is_empty(),+ "Swift's stop is silent on success; got stdout: {stdout2:?}");+ println!("stop OK (silent stdout, matches Swift)");++ // 3. status after stop — exit code 1, stderr "not running".+ let out3 = Command::new(&exe).arg("status").output().expect("status2");+ assert_eq!(out3.status.code(), Some(1), "status after stop should exit 1");+ let stderr3 = String::from_utf8_lossy(&out3.stderr);+ assert!(stderr3.contains("cua-driver daemon is not running"),+ "stderr should say 'not running': {stderr3:?}");+ println!("status-not-running OK");++ // 4. stop on dead daemon — exit 1, stderr "not running".+ let out4 = Command::new(&exe).arg("stop").output().expect("stop2");+ assert_eq!(out4.status.code(), Some(1), "stop on dead daemon should exit 1");+ let stderr4 = String::from_utf8_lossy(&out4.stderr);+ assert!(stderr4.contains("cua-driver daemon is not running"),+ "stderr should say 'not running': {stderr4:?}");+ println!("stop-not-running OK");++ println!("\n✅ PASS: status + stop output + exit codes match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/describe_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/describe_parity.rsnew file mode 100644index 0000000000..4164adab0a--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/describe_parity.rs@@ -0,0 +1,52 @@+//! Parity check for the `describe` CLI subcommand.++use std::process::Command;++#[cfg(target_os = "windows")]+fn main() {+ let exe = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .parent().unwrap()+ .parent().unwrap()+ .join("target/debug/cua-driver.exe");++ // 1. Known tool — describe `click`.+ let out = Command::new(&exe)+ .args(["describe", "click"])+ .output()+ .expect("run cua-driver describe click");+ assert!(out.status.success(), "describe click failed: {:?}", out);+ let stdout = String::from_utf8_lossy(&out.stdout);+ assert!(stdout.starts_with("name: click\n"),+ "describe should start with 'name: click\\n', got: {:?}", &stdout[..stdout.len().min(40)]);+ assert!(stdout.contains("\ndescription:\n"), "describe missing description: block");+ assert!(stdout.contains("\ninput_schema:\n"), "describe missing input_schema: block");+ // The JSON schema after should be pretty-printed (multi-line).+ let schema_section = stdout.split("\ninput_schema:\n").nth(1).unwrap_or("");+ assert!(schema_section.contains("\n "), "input_schema should be pretty-printed JSON");+ println!("describe known-tool OK");++ // 2. Unknown tool — exit code 64, stderr lists tools alphabetically.+ let out2 = Command::new(&exe)+ .args(["describe", "this_tool_does_not_exist"])+ .output()+ .expect("run cua-driver describe missing");+ assert_eq!(out2.status.code(), Some(64), "exit code should be 64 (EX_USAGE), got: {:?}", out2.status.code());+ let stderr = String::from_utf8_lossy(&out2.stderr);+ assert!(stderr.contains("Unknown tool: this_tool_does_not_exist"),+ "stderr missing 'Unknown tool:' line: {stderr:?}");+ assert!(stderr.contains("Available tools:"),+ "stderr missing 'Available tools:' header");+ // Parse the listed tools (each on its own indented line) and verify sort.+ let listed: Vec<&str> = stderr.lines()+ .filter(|l| l.starts_with(" ") && !l.trim().is_empty())+ .map(|l| l.trim())+ .collect();+ let mut sorted = listed.clone(); sorted.sort();+ assert_eq!(listed, sorted, "available tools should be sorted alphabetically");+ println!("describe unknown-tool OK ({} tools listed)", listed.len());++ println!("\n✅ PASS: describe known/unknown paths + Swift sort verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/double_click_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/double_click_parity.rsnew file mode 100644index 0000000000..d8ceece23c--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/double_click_parity.rs@@ -0,0 +1,97 @@+//! Parity check for `double_click` text format + error wording.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // Find a Chrome window to click on (same approach as click_parity).+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome") || n.contains("notepad")+ }).expect("no chrome/notepad");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();++ // 1. Missing target with valid pid+window_id.+ let r1 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"double_click","args":{{"pid":{pid},"window_id":{wid}}}}}"#));+ let v1: serde_json::Value = serde_json::from_str(r1.trim()).unwrap();+ let e1 = extract_text(&v1);+ assert!(e1.contains("Provide element_index or (x, y) to address the double-click target"),+ "Missing-target wording wrong: {e1:?}");+ println!("Missing-target err OK");++ // 2. Partial xy: x without y.+ let r2 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"double_click","args":{{"pid":{pid},"window_id":{wid},"x":100}}}}"#));+ let v2: serde_json::Value = serde_json::from_str(r2.trim()).unwrap();+ let e2 = extract_text(&v2);+ assert!(e2.contains("Provide both x and y together"),+ "Partial-xy wording wrong: {e2:?}");+ println!("Partial-xy err OK");++ // 3. Both element_index AND xy.+ let r3 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"double_click","args":{{"pid":{pid},"window_id":{wid},"element_index":0,"x":100,"y":100}}}}"#));+ let v3: serde_json::Value = serde_json::from_str(r3.trim()).unwrap();+ let e3 = extract_text(&v3);+ assert!(e3.contains("Provide either element_index or (x, y), not both"),+ "Both-modes wording wrong: {e3:?}");+ println!("Both-modes err OK");++ // 4. element_index without window_id.+ let r4 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"double_click","args":{{"pid":{pid},"element_index":0}}}}"#));+ let v4: serde_json::Value = serde_json::from_str(r4.trim()).unwrap();+ let e4 = extract_text(&v4);+ assert!(e4.contains("window_id is required when element_index is used"),+ "Element-without-window wording wrong: {e4:?}");+ println!("Element-without-window err OK");++ // 5. Real pixel double-click.+ let dc_req = format!(+ r#"{{"method":"call","name":"double_click","args":{{"pid":{pid},"window_id":{wid},"x":300,"y":300}}}}"#);+ let r5 = req(&mut pipe, &dc_req);+ let v5: serde_json::Value = serde_json::from_str(r5.trim()).unwrap();+ let t5 = extract_text(&v5);+ println!("Double-click resp: {t5:?}");+ assert!(t5.starts_with("✅ Posted double-click to pid ")+ && t5.contains(" at window-pixel (")+ && t5.contains(" → screen-point ("),+ "Pixel-path text doesn't match Swift format: {t5:?}");++ println!("\n✅ PASS: double_click validation + pixel-path text match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/drag_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/drag_parity.rsnew file mode 100644index 0000000000..4e6e0aa880--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/drag_parity.rs@@ -0,0 +1,67 @@+//! Parity check for `drag` error wording + text format.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(6);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome")+ }).expect("no chrome");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();++ // 1. Missing coordinates.+ let r1 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"drag","args":{{"pid":{pid},"window_id":{wid},"from_x":10}}}}"#));+ let e1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert!(e1.contains("from_x, from_y, to_x, and to_y are all required"),+ "Missing-coords wording wrong: {e1:?}");+ println!("Missing-coords err OK");++ // 2. Real drag — short, won't break Chrome.+ let r2 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"drag","args":{{"pid":{pid},"window_id":{wid},"from_x":100,"from_y":100,"to_x":102,"to_y":102,"duration_ms":50,"steps":2}}}}"#));+ let t2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ println!("Drag resp: {t2:?}");+ assert!(t2.starts_with("✅ Posted drag to pid ")+ && t2.contains("from window-pixel (")+ && t2.contains("screen (")+ && t2.contains("in 50ms / 2 steps."),+ "Drag text format wrong: {t2:?}");++ println!("\n✅ PASS: drag error wording + Swift-shape text format verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/dump_docs_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/dump_docs_parity.rsnew file mode 100644index 0000000000..008d25f4d2--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/dump_docs_parity.rs@@ -0,0 +1,49 @@+//! Parity check for `cua-driver dump-docs`.++use std::process::Command;++#[cfg(target_os = "windows")]+fn main() {+ let exe = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .parent().unwrap()+ .parent().unwrap()+ .join("target/debug/cua-driver.exe");++ // 1. Default: --type=all → {cli, mcp}.+ let out = Command::new(&exe).arg("dump-docs").output().expect("run");+ assert!(out.status.success());+ let stdout = String::from_utf8_lossy(&out.stdout);+ let v: serde_json::Value = serde_json::from_str(&stdout).expect("valid JSON");+ assert!(v.get("cli").is_some(), "--type=all should have 'cli' key");+ assert!(v.get("mcp").is_some(), "--type=all should have 'mcp' key");+ assert!(v.pointer("/mcp/version").is_some(), "mcp.version missing");+ let tools = v.pointer("/mcp/tools").and_then(|t| t.as_array()).expect("mcp.tools array");+ assert!(!tools.is_empty(), "mcp.tools should not be empty");+ println!("dump-docs (default) OK — {} tools", tools.len());++ // 2. --type=mcp → just {version, tools: [...]}.+ let out2 = Command::new(&exe).args(["dump-docs", "--type", "mcp"]).output().expect("run");+ let v2: serde_json::Value = serde_json::from_str(&String::from_utf8_lossy(&out2.stdout)).expect("valid JSON");+ assert!(v2.get("cli").is_none(), "--type=mcp should NOT have 'cli' key");+ assert!(v2.get("version").is_some(), "--type=mcp top-level should have version");+ assert!(v2.get("tools").is_some(), "--type=mcp top-level should have tools");+ println!("dump-docs --type=mcp OK");++ // 3. --type=cli → CLI section only (stub on Rust).+ let out3 = Command::new(&exe).args(["dump-docs", "--type", "cli"]).output().expect("run");+ let v3: serde_json::Value = serde_json::from_str(&String::from_utf8_lossy(&out3.stdout)).expect("valid JSON");+ assert!(v3.get("mcp").is_none(), "--type=cli should NOT have 'mcp' key");+ assert!(v3.get("version").is_some(), "--type=cli should have version");+ println!("dump-docs --type=cli OK");++ // 4. --pretty: should have multi-line output.+ let out4 = Command::new(&exe).args(["dump-docs", "--pretty"]).output().expect("run");+ let stdout4 = String::from_utf8_lossy(&out4.stdout);+ assert!(stdout4.lines().count() > 10, "--pretty should produce multi-line output");+ println!("dump-docs --pretty OK ({} lines)", stdout4.lines().count());++ println!("\n✅ PASS: dump-docs --type cli/mcp/all + --pretty all work");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/get_agent_cursor_state_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/get_agent_cursor_state_parity.rsnew file mode 100644index 0000000000..4dfd0c4d23--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/get_agent_cursor_state_parity.rs@@ -0,0 +1,61 @@+//! Parity check for `get_agent_cursor_state`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ let r = req(&mut pipe, r#"{"method":"call","name":"get_agent_cursor_state","args":{}}"#);+ let v: serde_json::Value = serde_json::from_str(r.trim()).unwrap();+ let text = v.pointer("/result/content/0/text").and_then(|t| t.as_str()).unwrap_or("");+ println!("State text: {text:?}");++ // Verify Swift's exact key=value vocabulary.+ let needed = [+ "✅ cursor: enabled=",+ "startHandle=",+ "endHandle=",+ "arcSize=",+ "arcFlow=",+ "spring=",+ "glideDurationMs=",+ "dwellAfterClickMs=",+ "idleHideMs=",+ ];+ for k in needed {+ assert!(text.contains(k), "Missing key {k:?} in text: {text:?}");+ }++ // Verify structuredContent.+ let sc = v.pointer("/result/structuredContent").unwrap();+ for k in &["enabled","start_handle","end_handle","arc_size","arc_flow","spring",+ "glide_duration_ms","dwell_after_click_ms","idle_hide_ms","cursors"] {+ assert!(sc.get(k).is_some(), "structuredContent missing {k}");+ }++ println!("\n✅ PASS: get_agent_cursor_state matches Swift vocabulary + structuredContent");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/get_cursor_position_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/get_cursor_position_parity.rsnew file mode 100644index 0000000000..bcd2fbde19--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/get_cursor_position_parity.rs@@ -0,0 +1,80 @@+//! Parity check for the `get_cursor_position` MCP tool.+//!+//! Drives the live daemon via the named pipe. Asserts the text response+//! matches Swift's format exactly: `"✅ Cursor at (X, Y)"` with integer+//! coordinates, and that `structuredContent` contains `x`/`y` as integers+//! that agree with the platform's native `GetCursorPos`.+//!+//! Hard 4-second timeout on every round-trip.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe — start daemon first");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 8192];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("response timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ let resp = req(&mut pipe, r#"{"method":"call","name":"get_cursor_position","args":{}}"#);+ let v: serde_json::Value = serde_json::from_str(resp.trim()).expect("parse JSON");++ let text = v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .expect("missing /result/content/0/text");+ println!("Response text: {text:?}");++ // Parse "✅ Cursor at (X, Y)" by hand — no regex crate dependency.+ let stripped = text.strip_prefix("✅ Cursor at (")+ .and_then(|s| s.strip_suffix(')'))+ .unwrap_or_else(|| panic!(+ "Text {text:?} does not match Swift format `✅ Cursor at (X, Y)`"+ ));+ let (xs, ys) = stripped.split_once(", ")+ .unwrap_or_else(|| panic!("missing `, ` between coords in {text:?}"));+ let tx: i64 = xs.parse().unwrap_or_else(|_| panic!("x not int: {xs}"));+ let ty: i64 = ys.parse().unwrap_or_else(|_| panic!("y not int: {ys}"));+ println!("Parsed text coords: ({tx}, {ty})");++ let sx = v.pointer("/result/structuredContent/x").and_then(|n| n.as_i64())+ .expect("structuredContent.x not present or not integer");+ let sy = v.pointer("/result/structuredContent/y").and_then(|n| n.as_i64())+ .expect("structuredContent.y not present or not integer");+ println!("Structured coords: ({sx}, {sy})");++ assert_eq!((tx, ty), (sx, sy), "text and structuredContent disagree");++ // Compare against the native GetCursorPos at this moment for a sanity check+ // (allow ±5 px slop since the cursor can drift between the daemon call+ // and our local call).+ use windows::Win32::Foundation::POINT;+ use windows::Win32::UI::WindowsAndMessaging::GetCursorPos;+ let mut pt = POINT { x: 0, y: 0 };+ unsafe { let _ = GetCursorPos(&mut pt); }+ let dx = (pt.x as i64 - sx).abs();+ let dy = (pt.y as i64 - sy).abs();+ println!("Native GetCursorPos: ({}, {}) — delta ({dx}, {dy})", pt.x, pt.y);+ assert!(dx <= 5 && dy <= 5,+ "get_cursor_position reports ({sx},{sy}) but native GetCursorPos says ({}, {})", pt.x, pt.y);++ println!("\n✅ PASS: text format matches Swift exactly, structuredContent agrees, native ≈ same");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/get_screen_size_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/get_screen_size_parity.rsnew file mode 100644index 0000000000..493d72218a--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/get_screen_size_parity.rs@@ -0,0 +1,80 @@+//! Parity check for `get_screen_size`.+//!+//! Asserts text matches Swift exactly:+//! `"✅ Main display: WxH points @ Sx"`+//! and `structuredContent` has `width`, `height`, and `scale_factor`+//! (snake_case) that agree with the platform's `GetSystemMetrics` and+//! `GetDpiForSystem`.+//!+//! Hard 4-second per-call timeout.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe — start daemon first");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 8192];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("response timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ let resp = req(&mut pipe, r#"{"method":"call","name":"get_screen_size","args":{}}"#);+ let v: serde_json::Value = serde_json::from_str(resp.trim()).expect("parse JSON");+ let text = v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .expect("missing text");+ println!("Response text: {text:?}");++ // Parse "✅ Main display: WxH points @ Sx" by hand.+ let rest = text.strip_prefix("✅ Main display: ")+ .unwrap_or_else(|| panic!("Text {text:?} missing Swift prefix `✅ Main display: `"));+ let (wh, scale_part) = rest.split_once(" points @ ")+ .unwrap_or_else(|| panic!("Text {text:?} missing ` points @ `"));+ let (w_str, h_str) = wh.split_once('x')+ .unwrap_or_else(|| panic!("Text {text:?} missing `x` separator"));+ let scale_str = scale_part.strip_suffix('x')+ .unwrap_or_else(|| panic!("Text {text:?} missing trailing `x`"));+ let tw: i64 = w_str.parse().unwrap();+ let th: i64 = h_str.parse().unwrap();+ let ts: f64 = scale_str.parse().unwrap();+ println!("Parsed text: width={tw} height={th} scale={ts}");++ // Verify structuredContent.+ let sw = v.pointer("/result/structuredContent/width").and_then(|n| n.as_i64()).expect("width");+ let sh = v.pointer("/result/structuredContent/height").and_then(|n| n.as_i64()).expect("height");+ let ss = v.pointer("/result/structuredContent/scale_factor").and_then(|n| n.as_f64()).expect("scale_factor");+ println!("Structured: width={sw} height={sh} scale_factor={ss}");++ assert_eq!((tw, th), (sw, sh));+ assert!((ts - ss).abs() < 0.001);++ // Sanity-check against native APIs.+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};+ use windows::Win32::UI::HiDpi::GetDpiForSystem;+ let (nw, nh) = unsafe { (GetSystemMetrics(SM_CXSCREEN) as i64, GetSystemMetrics(SM_CYSCREEN) as i64) };+ let dpi = unsafe { GetDpiForSystem() };+ let ns = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 };+ println!("Native: width={nw} height={nh} scale={ns}");+ assert_eq!((tw, th), (nw, nh));+ assert!((ts - ns).abs() < 0.001);++ println!("\n✅ PASS: text format matches Swift, structuredContent has scale_factor, native agrees");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/get_window_state_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/get_window_state_parity.rsnew file mode 100644index 0000000000..11736acd24--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/get_window_state_parity.rs@@ -0,0 +1,86 @@+//! Parity check for `get_window_state` error wording + validation guards.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(6);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. Missing pid.+ let r1 = req(&mut pipe, r#"{"method":"call","name":"get_window_state","args":{}}"#);+ let e1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert!(e1.to_lowercase().contains("pid") && e1.to_lowercase().contains("required"),+ "Missing-pid wording wrong: {e1:?}");+ println!("Missing-pid err OK");++ // 2. Missing window_id (with valid pid).+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome")+ }).expect("no chrome");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();++ let r2 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"get_window_state","args":{{"pid":{pid}}}}}"#));+ let e2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ assert!(e2.to_lowercase().contains("window_id") && e2.to_lowercase().contains("required"),+ "Missing-window_id wording wrong: {e2:?}");+ println!("Missing-window_id err OK");++ // 3. Mismatched pid + window_id (pick another window's pid).+ let other = wins.iter().find(|w| {+ let p = w.pointer("/pid").and_then(|n| n.as_u64()).unwrap_or(0);+ p != pid+ });+ if let Some(other) = other {+ let other_pid = other.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let r3 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"get_window_state","args":{{"pid":{other_pid},"window_id":{wid}}}}}"#));+ let e3 = extract_text(&serde_json::from_str(r3.trim()).unwrap());+ assert!(e3.contains("belongs to pid") || e3.contains("No window with window_id"),+ "Mismatched-pid wording wrong: {e3:?}");+ println!("Mismatched-pid err OK");+ }++ // 4. Bogus window_id.+ let r4 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"get_window_state","args":{{"pid":{pid},"window_id":99999999}}}}"#));+ let e4 = extract_text(&serde_json::from_str(r4.trim()).unwrap());+ assert!(e4.contains("No window with window_id"),+ "Bogus-window_id wording wrong: {e4:?}");+ println!("Bogus-window_id err OK");++ println!("\n✅ PASS: get_window_state error wording + validation guards verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/hotkey_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/hotkey_parity.rsnew file mode 100644index 0000000000..0d11bea57d--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/hotkey_parity.rs@@ -0,0 +1,75 @@+//! Parity check for `hotkey`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. Missing pid — daemon's schema validator catches this before the+ // tool runs; either schema-layer or tool-layer wording is acceptable+ // as long as `pid` is named in the error.+ let r1 = req(&mut pipe, r#"{"method":"call","name":"hotkey","args":{}}"#);+ let v1: serde_json::Value = serde_json::from_str(r1.trim()).unwrap();+ let e1 = extract_text(&v1);+ assert!(e1.to_lowercase().contains("pid") && e1.to_lowercase().contains("required"),+ "Missing-pid error should mention pid + required, got: {e1:?}");+ println!("Missing-pid err OK: {e1:?}");++ // 2. Missing keys.+ let r2 = req(&mut pipe, r#"{"method":"call","name":"hotkey","args":{"pid":1}}"#);+ let v2: serde_json::Value = serde_json::from_str(r2.trim()).unwrap();+ let e2 = extract_text(&v2);+ assert!(e2.to_lowercase().contains("keys") && e2.to_lowercase().contains("required"),+ "Missing-keys error should mention keys + required, got: {e2:?}");+ println!("Missing-keys err OK: {e2:?}");++ // 3. Real hotkey against Chrome — ctrl+end (scroll to bottom; harmless).+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome") || n.contains("notepad")+ }).expect("no target");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();+ let hk = format!(+ r#"{{"method":"call","name":"hotkey","args":{{"pid":{pid},"window_id":{wid},"keys":["ctrl","end"]}}}}"#);+ let r3 = req(&mut pipe, &hk);+ let v3: serde_json::Value = serde_json::from_str(r3.trim()).unwrap();+ let t3 = extract_text(&v3);+ println!("Hotkey resp: {t3:?}");+ assert!(t3.starts_with("✅ Pressed ctrl+end on pid "),+ "Expected Swift `✅ Pressed K1+K2 on pid X.`, got: {t3:?}");++ println!("\n✅ PASS: hotkey error wording + text format match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/launch_app_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/launch_app_parity.rsnew file mode 100644index 0000000000..38a4e882b1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/launch_app_parity.rs@@ -0,0 +1,87 @@+//! Parity check for `launch_app`.+//!+//! Launches Notepad (safe target) in background, asserts:+//! - Text starts with `"✅ Launched ... (pid <N>) in background."`+//! - structuredContent has `{pid, bundle_id, name, running, active, windows}`+//! - `active: false` (Swift's background-launch invariant)+//! - At least one entry in `windows` array OR `windows: []` if the window+//! didn't materialize within the 5×200ms retry budget.+//!+//! Cleans up: terminates the launched pid before exit.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe — start daemon first");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("response timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ // 1. Missing target error+ let r0 = req(&mut pipe, r#"{"method":"call","name":"launch_app","args":{}}"#);+ let v0: serde_json::Value = serde_json::from_str(r0.trim()).expect("parse");+ let err0 = v0.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v0.pointer("/error").and_then(|e| e.as_str()))+ .unwrap_or("");+ assert!(+ err0.contains("Provide either bundle_id or name to identify the app to launch"),+ "Missing-target error doesn't match Swift wording: {err0:?}"+ );+ println!("Missing-target err OK: {err0:?}");++ // 2. Launch Notepad+ let resp = req(&mut pipe, r#"{"method":"call","name":"launch_app","args":{"name":"notepad.exe"}}"#);+ let v: serde_json::Value = serde_json::from_str(resp.trim()).expect("parse");+ let text = v.pointer("/result/content/0/text").and_then(|t| t.as_str()).expect("missing text");+ println!("Launch resp text:\n{text}");++ let header = text.lines().next().unwrap_or("");+ assert!(+ header.starts_with("✅ Launched ") && header.contains(" (pid ") && header.ends_with("in background."),+ "Header {header:?} doesn't match Swift format `✅ Launched <name> (pid X) in background.`"+ );++ let pid = v.pointer("/result/structuredContent/pid").and_then(|n| n.as_u64()).expect("pid");+ let name = v.pointer("/result/structuredContent/name").and_then(|s| s.as_str()).expect("name");+ let running = v.pointer("/result/structuredContent/running").and_then(|b| b.as_bool()).expect("running");+ let active = v.pointer("/result/structuredContent/active").and_then(|b| b.as_bool()).expect("active");+ let windows = v.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).expect("windows");+ let bundle_id = v.pointer("/result/structuredContent/bundle_id").expect("bundle_id key");++ println!("Structured: pid={pid} name={name:?} running={running} active={active} windows={} bundle_id={bundle_id}", windows.len());++ assert!(pid > 0, "pid must be captured from ShellExecuteEx");+ assert!(running, "running must be true");+ assert!(!active, "active must be false (background launch — Swift's invariant)");+ assert!(bundle_id.is_null(), "bundle_id must be null on Windows");++ // Cleanup: kill notepad.+ std::process::Command::new("taskkill")+ .args(["/PID", &pid.to_string(), "/F"])+ .stdout(std::process::Stdio::null())+ .stderr(std::process::Stdio::null())+ .status().ok();++ println!("\n✅ PASS: launch_app text format, pid capture, and Swift invariants verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/list_apps_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/list_apps_parity.rsnew file mode 100644index 0000000000..6b4bc1eb8b--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/list_apps_parity.rs@@ -0,0 +1,76 @@+//! Parity check for `list_apps`.+//!+//! Asserts the response has Swift's shape:+//! - text begins with `"✅ Found N app(s): R running, I installed-not-running."`+//! - one bullet line per running app: `"- <name> (pid <N>)"`+//! - `structuredContent.apps` is an array of `{pid, bundle_id, name, running, active}`+//! - at least one app is marked `active: true` (something must own the+//! foreground window when this test runs)+//!+//! Hard 4-second timeout on the round-trip.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe — start daemon first");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("response timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ let resp = req(&mut pipe, r#"{"method":"call","name":"list_apps","args":{}}"#);+ let v: serde_json::Value = serde_json::from_str(resp.trim()).expect("parse JSON");++ let text = v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .expect("missing text");+ let first_line = text.lines().next().unwrap_or("");+ println!("Header: {first_line:?}");+ assert!(+ first_line.starts_with("✅ Found ") && first_line.contains(" app(s): "),+ "Header {first_line:?} does not match Swift `✅ Found N app(s): R running, I installed-not-running.`"+ );++ // structuredContent+ let apps = v.pointer("/result/structuredContent/apps").and_then(|a| a.as_array())+ .expect("structuredContent.apps is not an array");+ assert!(!apps.is_empty(), "Expected at least one running app");+ println!("Apps: {} entries", apps.len());++ let mut any_active = false;+ for (i, app) in apps.iter().enumerate() {+ let pid = app.pointer("/pid").and_then(|n| n.as_u64()).expect("pid");+ let name = app.pointer("/name").and_then(|s| s.as_str()).expect("name");+ let running = app.pointer("/running").and_then(|b| b.as_bool()).expect("running");+ let active = app.pointer("/active").and_then(|b| b.as_bool()).expect("active");+ // bundle_id present (null on Windows, optional string on macOS).+ let _ = app.pointer("/bundle_id").expect("bundle_id key missing");+ if i < 5 {+ println!(" [{i}] pid={pid} name={name:?} running={running} active={active}");+ }+ assert!(running, "Windows list_apps should only return running apps for now");+ if active { any_active = true; }+ }+ assert!(any_active, "Expected at least one app marked active (the foreground window owner)");++ println!("\n✅ PASS: text header + structuredContent.apps shape match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/list_tools_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/list_tools_parity.rsnew file mode 100644index 0000000000..9f5c913bb5--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/list_tools_parity.rs@@ -0,0 +1,57 @@+//! Parity check for the `list-tools` CLI subcommand.+//!+//! Runs the binary directly and asserts:+//! - Output is one tool per line as `"name: <first sentence>"` or just `"name"`.+//! - Lines are sorted alphabetically by tool name (matches Swift's sort).+//! - All expected core tools are present.++use std::process::Command;++#[cfg(target_os = "windows")]+fn main() {+ let exe = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .parent().unwrap()+ .parent().unwrap()+ .join("target/debug/cua-driver.exe");+ let out = Command::new(&exe)+ .arg("list-tools")+ .output()+ .expect("run cua-driver list-tools");+ assert!(out.status.success(), "list-tools failed: {:?}", out);+ let stdout = String::from_utf8_lossy(&out.stdout);+ let lines: Vec<&str> = stdout.lines().collect();+ assert!(!lines.is_empty(), "no output from list-tools");++ // Names are sorted ascending.+ let names: Vec<&str> = lines.iter().map(|l| l.split(':').next().unwrap()).collect();+ let mut sorted = names.clone();+ sorted.sort();+ assert_eq!(names, sorted, "tools are not sorted alphabetically (Swift parity)");++ // All core tools are present.+ let expected = [+ "click", "double_click", "right_click", "type_text", "press_key", "hotkey",+ "scroll", "screenshot", "list_apps", "list_windows", "get_cursor_position",+ "get_screen_size", "launch_app", "move_cursor", "set_agent_cursor_enabled",+ "set_agent_cursor_motion", "get_agent_cursor_state", "set_value",+ "get_config", "set_config", "check_permissions", "get_recording_state",+ "set_recording",+ ];+ for tool in expected {+ assert!(names.iter().any(|n| *n == tool), "missing tool `{tool}` in list-tools output");+ }++ // Each non-empty line should have `name: <summary>` shape OR just `name`.+ for line in &lines {+ if line.contains(':') {+ let after = line.split_once(':').unwrap().1;+ assert!(after.trim_start().chars().next().is_some(),+ "empty description after colon in line: {line:?}");+ }+ }++ println!("\n✅ PASS: list-tools is alphabetically sorted, all core tools present, format matches Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/list_windows_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/list_windows_parity.rsnew file mode 100644index 0000000000..d3cb10bbf1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/list_windows_parity.rs@@ -0,0 +1,78 @@+//! Parity check for `list_windows`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe — start daemon first");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("response timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ let resp = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let v: serde_json::Value = serde_json::from_str(resp.trim()).expect("parse JSON");+ let text = v.pointer("/result/content/0/text").and_then(|t| t.as_str()).expect("missing text");+ let header = text.lines().next().unwrap_or("");+ println!("Header: {header:?}");+ assert!(+ header.starts_with("✅ Found ") && header.contains(" window(s) across "),+ "Header does not match Swift format: {header:?}"+ );+ assert!(+ header.contains("SkyLight Space SPIs unavailable"),+ "Windows should append the SPI-unavailable note (matches Swift's else branch)"+ );++ let wins = v.pointer("/result/structuredContent/windows").and_then(|w| w.as_array())+ .expect("structuredContent.windows is not an array");+ assert!(!wins.is_empty(), "expected at least one visible window");++ for w in wins {+ for key in &["window_id","pid","app_name","title","bounds","layer","z_index","is_on_screen"] {+ assert!(w.pointer(&format!("/{key}")).is_some(),+ "window record missing field {key}: {w}");+ }+ // bounds shape+ for k in &["x","y","width","height"] {+ assert!(w.pointer(&format!("/bounds/{k}")).is_some(), "bounds missing {k}");+ }+ // Swift omits these on Windows (SPI unavailable); explicitly assert+ // they are NOT present.+ assert!(w.get("on_current_space").is_none(), "on_current_space should be omitted on Windows");+ assert!(w.get("space_ids").is_none(), "space_ids should be omitted on Windows");+ }++ let csi = v.pointer("/result/structuredContent/current_space_id");+ assert!(csi.is_some() && csi.unwrap().is_null(),+ "current_space_id should be null on Windows; got {:?}", csi);++ // Pid-filter warning path.+ let resp2 = req(&mut pipe,+ r#"{"method":"call","name":"list_windows","args":{"pid":4294967295}}"#);+ let v2: serde_json::Value = serde_json::from_str(resp2.trim()).expect("parse JSON");+ let t2 = v2.pointer("/result/content/0/text").and_then(|t| t.as_str()).unwrap_or("");+ assert!(t2.starts_with("⚠️ No windows found for pid"),+ "Expected ⚠️-prefixed warning for bogus pid, got: {t2:?}");++ println!("\n✅ PASS: text header, structuredContent fields + bounds shape, and pid-warning path all match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/mcp_config_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/mcp_config_parity.rsnew file mode 100644index 0000000000..4bee36f68c--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/mcp_config_parity.rs@@ -0,0 +1,49 @@+//! Parity check for the `mcp-config` CLI subcommand.++use std::process::Command;++#[cfg(target_os = "windows")]+fn main() {+ let exe = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .parent().unwrap()+ .parent().unwrap()+ .join("target/debug/cua-driver.exe");++ let cases = [+ // Each (args, expected substrings)+ (vec![], vec!["\"mcpServers\"", "\"cua-driver\"", "\"command\":", "\"args\": [\"mcp\"]"]),+ (vec!["--client", "claude"], vec!["claude mcp add --transport stdio cua-driver"]),+ (vec!["--client", "codex"], vec!["codex mcp add cua-driver"]),+ (vec!["--client", "cursor"], vec!["\"type\": \"stdio\"", "\"mcpServers\""]),+ (vec!["--client", "openclaw"], vec!["openclaw mcp set cua-driver"]),+ (vec!["--client", "opencode"], vec!["opencode.ai/config.json", "\"type\": \"local\""]),+ (vec!["--client", "hermes"], vec!["mcp_servers:", "cua-driver:"]),+ (vec!["--client", "pi"], vec!["does not support MCP natively"]),+ ];++ for (args, needles) in cases {+ let mut cmd_args = vec!["mcp-config"];+ cmd_args.extend(args.iter().copied());+ let out = Command::new(&exe).args(&cmd_args).output().expect("run");+ assert!(out.status.success(), "mcp-config {args:?} failed: {out:?}");+ let stdout = String::from_utf8_lossy(&out.stdout);+ for needle in &needles {+ assert!(stdout.contains(needle),+ "mcp-config {args:?} output missing {needle:?}\n--- stdout ---\n{stdout}");+ }+ println!("mcp-config {args:?} OK");+ }++ // Unknown client → exit 2, stderr names valid clients.+ let out = Command::new(&exe).args(["mcp-config", "--client", "bogus"]).output().expect("run");+ assert_eq!(out.status.code(), Some(2), "unknown client should exit 2");+ let stderr = String::from_utf8_lossy(&out.stderr);+ assert!(stderr.contains("Unknown client 'bogus'") && stderr.contains("claude"),+ "unknown-client stderr wrong: {stderr:?}");+ println!("unknown-client err OK");++ println!("\n✅ PASS: mcp-config 8 client paths + unknown-client error verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/overlay_dump.rs b/libs/cua-driver-rs/crates/platform-windows/examples/overlay_dump.rsnew file mode 100644index 0000000000..ed8397330f--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/overlay_dump.rs@@ -0,0 +1,107 @@+//! Capture the daemon's overlay window content directly via PrintWindow.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ use windows::Win32::Foundation::HWND;+ use windows::Win32::Graphics::Gdi::*;+ use windows::Win32::UI::WindowsAndMessaging::*;+ use std::ffi::OsStr;+ use std::os::windows::ffi::OsStrExt;++ // Send the daemon a move command first.+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");+ fn req(p: &mut std::fs::File, json: &str) {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut buf = [0u8; 8192];+ let deadline = Instant::now() + Duration::from_secs(2);+ let mut out = Vec::new();+ loop {+ if Instant::now() > deadline { break; }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ }+ req(&mut pipe, r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}"#);+ req(&mut pipe, r#"{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"set_agent_cursor_enabled","arguments":{"enabled":true}}}"#);+ req(&mut pipe, r#"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"move_cursor","arguments":{"x":700.0,"y":500.0}}}"#);++ std::thread::sleep(Duration::from_millis(1500));++ // Find overlay HWND.+ let class_name: Vec<u16> = OsStr::new("TropeCUA.AgentCursorOverlay\0")+ .encode_wide().collect();+ let hwnd = unsafe { FindWindowW(+ windows::core::PCWSTR(class_name.as_ptr()),+ windows::core::PCWSTR::null(),+ ) };+ let hwnd = match hwnd { Ok(h) => h, Err(_) => { eprintln!("no overlay window"); std::process::exit(1); } };+ println!("overlay HWND: 0x{:X}", hwnd.0 as isize);++ let mut rect = windows::Win32::Foundation::RECT::default();+ unsafe { let _ = GetWindowRect(hwnd, &mut rect); }+ let w = rect.right - rect.left;+ let h = rect.bottom - rect.top;+ println!("rect: {},{} {}x{}", rect.left, rect.top, w, h);++ // PrintWindow capture.+ unsafe {+ let screen_dc = GetDC(HWND::default());+ let mem_dc = CreateCompatibleDC(screen_dc);+ let bitmap = CreateCompatibleBitmap(screen_dc, w, h);+ let old = SelectObject(mem_dc, bitmap);+ // PW_RENDERFULLCONTENT = 0x2 — needed for layered windows+ let ok = PrintWindow(hwnd, mem_dc, PRINT_WINDOW_FLAGS(0x2));+ println!("PrintWindow returned: {}", ok.as_bool());++ let mut bmi = BITMAPINFO {+ bmiHeader: BITMAPINFOHEADER {+ biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,+ biWidth: w, biHeight: -h, biPlanes: 1, biBitCount: 32,+ biCompression: BI_RGB.0,+ biSizeImage: (w * h * 4) as u32,+ ..Default::default()+ },+ bmiColors: [RGBQUAD::default(); 1],+ };+ let mut pixels = vec![0u8; (w * h * 4) as usize];+ let got = GetDIBits(mem_dc, bitmap, 0, h as u32, Some(pixels.as_mut_ptr() as *mut _), &mut bmi, DIB_RGB_COLORS);+ println!("GetDIBits scanlines: {}", got);++ // Find non-zero pixels.+ let mut nonzero = 0u64;+ let mut sum_x = 0i64; let mut sum_y = 0i64;+ let mut max_a = 0u8;+ for y in 0..h as usize {+ for x in 0..w as usize {+ let i = (y * w as usize + x) * 4;+ let b = pixels[i]; let g = pixels[i+1]; let r = pixels[i+2]; let a = pixels[i+3];+ max_a = max_a.max(a);+ if r as u32 + g as u32 + b as u32 > 30 {+ nonzero += 1;+ sum_x += x as i64; sum_y += y as i64;+ }+ }+ }+ println!("non-zero RGB pixels in overlay surface: {} max_alpha={}", nonzero, max_a);+ if nonzero > 0 {+ println!("centroid: ({}, {})", sum_x / nonzero as i64, sum_y / nonzero as i64);+ }++ SelectObject(mem_dc, old);+ let _ = DeleteObject(bitmap);+ let _ = DeleteDC(mem_dc);+ ReleaseDC(HWND::default(), screen_dc);+ }+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/press_key_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/press_key_parity.rsnew file mode 100644index 0000000000..0d87fbace5--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/press_key_parity.rs@@ -0,0 +1,76 @@+//! Parity check for `press_key`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned())+ .unwrap_or_else(|| format!("(no text: {v})"))+ }++ // 1. Missing key error.+ let r1 = req(&mut pipe, r#"{"method":"call","name":"press_key","args":{"pid":1}}"#);+ let v1: serde_json::Value = serde_json::from_str(r1.trim()).unwrap();+ let err1 = extract_text(&v1);+ assert!(err1.contains("Missing required string field key"),+ "Expected Swift-style key error, got: {err1:?}");+ println!("Missing-key err OK: {err1:?}");++ // 2. element_index without window_id should error per Swift.+ let r2 = req(&mut pipe, r#"{"method":"call","name":"press_key","args":{"pid":1,"key":"a","element_index":0}}"#);+ let v2: serde_json::Value = serde_json::from_str(r2.trim()).unwrap();+ let err2 = extract_text(&v2);+ assert!(err2.contains("window_id is required when element_index is used"),+ "Expected Swift-style element-without-window error, got: {err2:?}");+ println!("Element-without-window err OK");++ // 3. Press a key against a real window.+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let preferred = ["chrome.exe", "notepad.exe"];+ let target = wins.iter().find(|w| {+ let name = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ preferred.iter().any(|p| name.contains(p))+ }).expect("no chrome/notepad");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();+ // Press End key (safe — scrolls to bottom of page; doesn't modify content).+ let press_req = format!(+ r#"{{"method":"call","name":"press_key","args":{{"pid":{pid},"window_id":{wid},"key":"end"}}}}"#);+ let r3 = req(&mut pipe, &press_req);+ let v3: serde_json::Value = serde_json::from_str(r3.trim()).unwrap();+ let text3 = extract_text(&v3);+ println!("Press resp: {text3:?}");+ assert!(text3.starts_with("✅ Pressed end on pid "),+ "Expected Swift format `✅ Pressed KEY on pid X.`, got: {text3:?}");++ println!("\n✅ PASS: press_key error wording + text format match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/recording_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/recording_parity.rsnew file mode 100644index 0000000000..0150374cf6--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/recording_parity.rs@@ -0,0 +1,83 @@+//! Parity check for `set_recording` + `get_recording_state`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. Disable to start clean.+ let _ = req(&mut pipe, r#"{"method":"call","name":"set_recording","args":{"enabled":false}}"#);++ // 2. get_recording_state — disabled.+ let r1 = req(&mut pipe, r#"{"method":"call","name":"get_recording_state","args":{}}"#);+ let t1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert_eq!(t1, "✅ recording: disabled", "Disabled text wrong: {t1:?}");+ println!("get_recording_state disabled OK");++ // 3. set_recording missing field.+ let r2 = req(&mut pipe, r#"{"method":"call","name":"set_recording","args":{}}"#);+ let e2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ assert!(e2.to_lowercase().contains("enabled") && e2.to_lowercase().contains("required"),+ "Missing-enabled wording wrong: {e2:?}");+ println!("Missing-enabled err OK");++ // 4. set_recording enabled without output_dir.+ let r3 = req(&mut pipe, r#"{"method":"call","name":"set_recording","args":{"enabled":true}}"#);+ let e3 = extract_text(&serde_json::from_str(r3.trim()).unwrap());+ assert!(e3.contains("`output_dir` is required when enabling recording"),+ "Missing-output_dir wording wrong: {e3:?}");+ println!("Missing-output_dir err OK");++ // 5. Enable with output_dir.+ let dir = std::env::temp_dir().join("cua-rec-parity").to_string_lossy().into_owned()+ .replace('\\', "\\\\");+ let r4 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"set_recording","args":{{"enabled":true,"output_dir":"{dir}"}}}}"#));+ let t4 = extract_text(&serde_json::from_str(r4.trim()).unwrap());+ println!("Enable resp: {t4:?}");+ assert!(t4.starts_with("✅ Recording enabled -> "),+ "Enable text wrong: {t4:?}");++ // 6. get_recording_state — enabled.+ let r5 = req(&mut pipe, r#"{"method":"call","name":"get_recording_state","args":{}}"#);+ let t5 = extract_text(&serde_json::from_str(r5.trim()).unwrap());+ println!("State enabled: {t5:?}");+ assert!(t5.starts_with("✅ recording: enabled output_dir=") && t5.contains("next_turn="),+ "Enabled state text wrong: {t5:?}");++ // 7. Disable.+ let r6 = req(&mut pipe, r#"{"method":"call","name":"set_recording","args":{"enabled":false}}"#);+ let t6 = extract_text(&serde_json::from_str(r6.trim()).unwrap());+ assert_eq!(t6, "✅ Recording disabled.", "Disabled text wrong: {t6:?}");++ println!("\n✅ PASS: get_recording_state + set_recording match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/replay_trajectory_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/replay_trajectory_parity.rsnew file mode 100644index 0000000000..8c09fa6ea4--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/replay_trajectory_parity.rs@@ -0,0 +1,52 @@+//! Parity check for `replay_trajectory` error wording.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. Empty `dir` string (schema-layer required, but should also be checked at tool layer).+ let r1 = req(&mut pipe, r#"{"method":"call","name":"replay_trajectory","args":{"dir":""}}"#);+ let e1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert!(e1.contains("`dir`") && e1.to_lowercase().contains("required"),+ "Empty-dir wording wrong: {e1:?}");+ println!("Empty-dir err OK");++ // 2. Nonexistent directory.+ let r2 = req(&mut pipe, r#"{"method":"call","name":"replay_trajectory","args":{"dir":"C:\\does\\not\\exist"}}"#);+ let e2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ assert!(e2.contains("Trajectory directory does not exist"),+ "Nonexistent-dir wording wrong: {e2:?}");+ println!("Nonexistent-dir err OK");++ println!("\n✅ PASS: replay_trajectory error paths match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/right_click_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/right_click_parity.rsnew file mode 100644index 0000000000..a2edcb9b4a--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/right_click_parity.rs@@ -0,0 +1,85 @@+//! Parity check for `right_click` text format + validation guards.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf = [0u8; 65536];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome") || n.contains("notepad")+ }).expect("no target");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();++ // 1. Missing target.+ let r1 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"right_click","args":{{"pid":{pid},"window_id":{wid}}}}}"#));+ let v1: serde_json::Value = serde_json::from_str(r1.trim()).unwrap();+ assert!(extract_text(&v1).contains("Provide element_index or (x, y) to address the right-click target"));+ println!("Missing-target err OK");++ // 2. Partial xy.+ let r2 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"right_click","args":{{"pid":{pid},"window_id":{wid},"x":50}}}}"#));+ assert!(extract_text(&serde_json::from_str(r2.trim()).unwrap()).contains("Provide both x and y together"));+ println!("Partial-xy err OK");++ // 3. Both modes.+ let r3 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"right_click","args":{{"pid":{pid},"window_id":{wid},"element_index":0,"x":50,"y":50}}}}"#));+ assert!(extract_text(&serde_json::from_str(r3.trim()).unwrap()).contains("Provide either element_index or (x, y), not both"));+ println!("Both-modes err OK");++ // 4. element_index without window_id.+ let r4 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"right_click","args":{{"pid":{pid},"element_index":0}}}}"#));+ assert!(extract_text(&serde_json::from_str(r4.trim()).unwrap()).contains("window_id is required when element_index is used"));+ println!("Element-without-window err OK");++ // 5. Real pixel right-click.+ let rc = format!(+ r#"{{"method":"call","name":"right_click","args":{{"pid":{pid},"window_id":{wid},"x":300,"y":300}}}}"#);+ let r5 = req(&mut pipe, &rc);+ let v5: serde_json::Value = serde_json::from_str(r5.trim()).unwrap();+ let t5 = extract_text(&v5);+ println!("Right-click resp: {t5:?}");+ assert!(t5.starts_with("✅ Posted right-click to pid ")+ && t5.contains(" at window-pixel (")+ && t5.contains(" → screen-point ("),+ "Pixel-path text doesn't match Swift format: {t5:?}");++ println!("\n✅ PASS: right_click validation + pixel-path text match Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/screenshot_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/screenshot_parity.rsnew file mode 100644index 0000000000..a7b56b99dd--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/screenshot_parity.rs@@ -0,0 +1,77 @@+//! Parity check for `screenshot`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ // Heap-allocate the 64 KiB read buffer — a stack array of this size+ // overflows the default thread stack on Windows debug builds.+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }++ // Window screenshot: find Chrome.+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome")+ }).expect("no chrome");+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();++ // 1. Window screenshot.+ let ss = format!(r#"{{"method":"call","name":"screenshot","args":{{"window_id":{wid}}}}}"#);+ let r1 = req(&mut pipe, &ss);+ let v1: serde_json::Value = serde_json::from_str(r1.trim()).unwrap();+ // Last content entry is text; first should be image.+ let content = v1.pointer("/result/content").and_then(|c| c.as_array()).unwrap();+ assert!(content.iter().any(|c| c.get("type").and_then(|t| t.as_str()) == Some("image")),+ "Response should contain an image content block");+ let text = content.iter().find_map(|c| {+ if c.get("type").and_then(|t| t.as_str()) == Some("text") {+ c.get("text").and_then(|t| t.as_str())+ } else { None }+ }).unwrap_or("");+ println!("Window resp: {text:?}");+ assert!(text.starts_with("✅ Window screenshot — ") && text.contains(&format!("[window_id: {wid}]")),+ "Window text doesn't match Swift format: {text:?}");+ let sc = v1.pointer("/result/structuredContent").unwrap();+ assert!(sc.get("width").is_some() && sc.get("height").is_some() && sc.get("format").is_some(),+ "structuredContent missing width/height/format");++ // 2. JPEG format.+ let ss2 = format!(+ r#"{{"method":"call","name":"screenshot","args":{{"window_id":{wid},"format":"jpeg","quality":50}}}}"#);+ let r2 = req(&mut pipe, &ss2);+ let v2: serde_json::Value = serde_json::from_str(r2.trim()).unwrap();+ let c2 = v2.pointer("/result/content").and_then(|c| c.as_array()).unwrap();+ let img = c2.iter().find(|c| c.get("type").and_then(|t| t.as_str()) == Some("image")).unwrap();+ let mime = img.get("mimeType").and_then(|m| m.as_str()).unwrap_or("");+ assert_eq!(mime, "image/jpeg", "jpeg format should set mimeType image/jpeg, got: {mime:?}");+ let fmt = v2.pointer("/result/structuredContent/format").and_then(|f| f.as_str()).unwrap_or("");+ assert_eq!(fmt, "jpeg", "structuredContent.format should be jpeg");++ println!("\n✅ PASS: screenshot text format + image + structuredContent + jpeg path verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/scroll_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/scroll_parity.rsnew file mode 100644index 0000000000..450a4a5400--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/scroll_parity.rs@@ -0,0 +1,70 @@+//! Parity check for `scroll`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // Missing direction error.+ let r1 = req(&mut pipe, r#"{"method":"call","name":"scroll","args":{"pid":1}}"#);+ let e1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert!(e1.contains("direction") && e1.to_lowercase().contains("required"),+ "Missing-direction error should mention direction + required: {e1:?}");+ println!("Missing-direction err OK");++ // element_index without window_id.+ let r2 = req(&mut pipe, r#"{"method":"call","name":"scroll","args":{"pid":1,"direction":"down","element_index":0}}"#);+ let e2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ assert!(e2.contains("window_id is required when element_index is used"),+ "Element-without-window wording wrong: {e2:?}");+ println!("Element-without-window err OK");++ // Real scroll against Chrome.+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome") || n.contains("notepad")+ }).expect("no target");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();+ let sc = format!(+ r#"{{"method":"call","name":"scroll","args":{{"pid":{pid},"window_id":{wid},"direction":"down","by":"page","amount":2}}}}"#);+ let r3 = req(&mut pipe, &sc);+ let t3 = extract_text(&serde_json::from_str(r3.trim()).unwrap());+ println!("Scroll resp: {t3:?}");+ assert!(t3.starts_with(&format!("✅ Scrolled pid {pid} down via 2× SB_PAGEDOWN message(s)")),+ "Scroll text doesn't match Swift-shape format: {t3:?}");++ println!("\n✅ PASS: scroll validation + Swift-shape text format verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/set_agent_cursor_style_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/set_agent_cursor_style_parity.rsnew file mode 100644index 0000000000..6f0c1a23f1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/set_agent_cursor_style_parity.rs@@ -0,0 +1,57 @@+//! Parity check for `set_agent_cursor_style`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. Set gradient + bloom — text should reflect both.+ let r1 = req(&mut pipe,+ r##"{"method":"call","name":"set_agent_cursor_style","args":{"gradient_colors":["#FF6B6B","#FF8E53"],"bloom_color":"#A855F7"}}"##);+ let t1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ println!("Style resp: {t1:?}");+ assert!(t1.contains("✅ cursor style:") && t1.contains("gradient_colors=[#FF6B6B,#FF8E53]")+ && t1.contains("bloom_color=#A855F7"),+ "Set-style text doesn't match Swift format: {t1:?}");+ println!("Set OK");++ // 2. Revert all — should produce "✅ cursor style: reverted to default".+ let r2 = req(&mut pipe,+ r#"{"method":"call","name":"set_agent_cursor_style","args":{"gradient_colors":[],"bloom_color":"","image_path":""}}"#);+ let t2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ println!("Revert resp: {t2:?}");+ assert_eq!(t2, "✅ cursor style: reverted to default",+ "Revert text wrong: {t2:?}");+ println!("Revert OK");++ println!("\n✅ PASS: set_agent_cursor_style text format matches Swift");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/set_value_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/set_value_parity.rsnew file mode 100644index 0000000000..1cd50b37fa--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/set_value_parity.rs@@ -0,0 +1,75 @@+//! Parity check for `set_value`.+//!+//! Tests the Swift-shaped error wording and text format. We don't issue a+//! real `set_value` call against a live element because that requires a+//! freshly-walked UIA cache; the response-shape verification on the error+//! paths is sufficient to lock the parity.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // Missing value error (with all three integers present). Note: pid /+ // window_id / element_index are schema-required; we pick a real pid ++ // window_id to ensure the schema doesn't reject before our tool runs.+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome") || n.contains("notepad")+ }).expect("no target");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();++ // schema layer requires `value` — daemon rejects with its own wording;+ // either Swift's tool-layer wording or schema-layer wording is acceptable+ // as long as `value` is named.+ let r1 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"set_value","args":{{"pid":{pid},"window_id":{wid},"element_index":0}}}}"#));+ let e1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert!(e1.to_lowercase().contains("value") && e1.to_lowercase().contains("required"),+ "Missing-value wording wrong: {e1:?}");+ println!("Missing-value err OK: {e1:?}");++ // Element-not-in-cache (we haven't run get_window_state). Should fail+ // with an "Element X not in cache" wording, not a schema error.+ let r2 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"set_value","args":{{"pid":{pid},"window_id":{wid},"element_index":99999,"value":"x"}}}}"#));+ let e2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ assert!(e2.to_lowercase().contains("not in cache") || e2.to_lowercase().contains("element"),+ "Cache-miss wording should mention element/cache: {e2:?}");+ println!("Cache-miss err OK: {e2:?}");++ println!("\n✅ PASS: set_value validation + Swift-shape error wording verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/type_text_chars_alias_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/type_text_chars_alias_parity.rsnew file mode 100644index 0000000000..6c0f91a3ca--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/type_text_chars_alias_parity.rs@@ -0,0 +1,72 @@+//! Parity check for the `type_text_chars` → `type_text` deprecated alias.+//!+//! Swift's ToolRegistry.swift resolves `type_text_chars` to `type_text` at+//! invoke time AND emits a stderr deprecation warning. The aliased name+//! is NOT in `tools/list`. Rust now does the same.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. `tools/list` (or equivalent) should NOT include `type_text_chars`.+ // The daemon's pipe has a `list` method that returns all registered+ // tools — we look for the alias and assert it's absent.+ let r1 = req(&mut pipe, r#"{"method":"list"}"#);+ println!("list resp (first 1000 chars): {:?}", r1.chars().take(1000).collect::<String>());+ assert!(!r1.contains("\"type_text_chars\""),+ "type_text_chars should NOT appear in tools/list (Swift alias convention)");+ assert!(r1.contains("\"type_text\""),+ "type_text should appear in tools/list");+ println!("tools/list excludes type_text_chars ✓");++ // 2. Invoking `type_text_chars` should still work — resolves to `type_text`.+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome") || n.contains("notepad")+ }).expect("no target");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();+ let r2 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"type_text_chars","args":{{"pid":{pid},"window_id":{wid},"text":"x"}}}}"#));+ let t2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ println!("type_text_chars resp: {t2:?}");+ // Should produce the same response as `type_text` (which uses+ // "✅ Typed N char(s) on pid X via PostMessage (Yms delay).").+ assert!(t2.starts_with("✅ Typed 1 char(s) on pid "),+ "type_text_chars should resolve to type_text response: {t2:?}");++ println!("\n✅ PASS: type_text_chars excluded from list AND aliased to type_text on invoke");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/type_text_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/type_text_parity.rsnew file mode 100644index 0000000000..09b3afe6fa--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/type_text_parity.rs@@ -0,0 +1,74 @@+//! Parity check for `type_text`.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ // 1. Missing text error.+ let r1 = req(&mut pipe, r#"{"method":"call","name":"type_text","args":{"pid":1}}"#);+ let e1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert!(e1.contains("text") && e1.to_lowercase().contains("required"),+ "Missing-text wording wrong: {e1:?}");+ println!("Missing-text err OK");++ // 2. element_index without window_id.+ let r2 = req(&mut pipe, r#"{"method":"call","name":"type_text","args":{"pid":1,"text":"x","element_index":0}}"#);+ let e2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ assert!(e2.contains("window_id is required when element_index is used"),+ "Element-without-window wording wrong: {e2:?}");+ println!("Element-without-window err OK");++ // 3. Real type against Chrome address bar — we can't reliably target it+ // here; just send to Chrome's window and verify the response text format.+ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome") || n.contains("notepad")+ }).expect("no target");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();+ let tt = format!(+ r#"{{"method":"call","name":"type_text","args":{{"pid":{pid},"window_id":{wid},"text":"hello","delay_ms":10}}}}"#);+ let r3 = req(&mut pipe, &tt);+ let t3 = extract_text(&serde_json::from_str(r3.trim()).unwrap());+ println!("Type resp: {t3:?}");+ // Swift format: "✅ Typed N char(s) on pid X via CGEvent (Yms delay)."+ // Windows variant: "✅ Typed N char(s) on pid X via PostMessage (Yms delay)."+ assert!(t3.starts_with(&format!("✅ Typed 5 char(s) on pid {pid} via PostMessage"))+ && t3.contains("10ms delay"),+ "Type text response doesn't match Swift-shape format: {t3:?}");++ println!("\n✅ PASS: type_text validation + Swift-shape text format verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/update_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/update_parity.rsnew file mode 100644index 0000000000..8221b76378--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/update_parity.rs@@ -0,0 +1,48 @@+//! Parity check for `cua-driver update` (no --apply, network-dependent).++use std::process::Command;++#[cfg(target_os = "windows")]+fn main() {+ let exe = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .parent().unwrap()+ .parent().unwrap()+ .join("target/debug/cua-driver.exe");++ let out = Command::new(&exe).arg("update").output().expect("run update");+ let stdout = String::from_utf8_lossy(&out.stdout);+ println!("update stdout (first 500 chars):");+ println!("{}", stdout.chars().take(500).collect::<String>());++ // Always-present lines.+ assert!(stdout.contains("Current version:"),+ "missing 'Current version:' header");+ assert!(stdout.contains("Checking for updates"),+ "missing 'Checking for updates' line");++ // Outcome is one of:+ // - "Already up to date." (current ≥ latest published)+ // - "New version available:" (current < latest published)+ // - "Could not reach GitHub" (network failure)+ let outcome_present = stdout.contains("Already up to date")+ || stdout.contains("New version available:")+ || stdout.contains("Could not reach GitHub");+ assert!(outcome_present, "no outcome line found in update output:\n{stdout}");++ // Critical: if a new version IS reported, the tag prefix must be the Rust-+ // port one (`cua-driver-rs-v*`) — otherwise we'd be recommending the+ // Swift binary's version which would be a real-user bug.+ if stdout.contains("New version available:") {+ let line = stdout.lines().find(|l| l.contains("New version available:")).unwrap();+ // The version printed should be numeric (e.g. "0.1.2"), not a+ // Swift-style version that exists on the same repo but is for a+ // different binary. We can't directly tell from the output, but+ // we can run with --apply mocked off (which we already are: no flag).+ println!("Found new version line: {line}");+ }++ println!("\n✅ PASS: update CLI prints expected outcome lines");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/examples/zoom_parity.rs b/libs/cua-driver-rs/crates/platform-windows/examples/zoom_parity.rsnew file mode 100644index 0000000000..01ca66f25b--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/examples/zoom_parity.rs@@ -0,0 +1,81 @@+//! Parity check for `zoom` error wording.++use std::io::{Read, Write};+use std::time::{Duration, Instant};++#[cfg(target_os = "windows")]+fn main() {+ let mut pipe = std::fs::OpenOptions::new()+ .read(true).write(true)+ .open(r"\\.\pipe\cua-driver")+ .expect("open pipe");++ fn req(p: &mut std::fs::File, json: &str) -> String {+ p.write_all(format!("{json}\n").as_bytes()).unwrap();+ p.flush().ok();+ let mut out = Vec::new();+ let mut buf: Vec<u8> = vec![0u8; 64 * 1024];+ let deadline = Instant::now() + Duration::from_secs(4);+ loop {+ if Instant::now() > deadline { panic!("timeout"); }+ let n = p.read(&mut buf).unwrap_or(0);+ if n == 0 { break; }+ out.extend_from_slice(&buf[..n]);+ if out.contains(&b'\n') { break; }+ }+ String::from_utf8_lossy(&out).into_owned()+ }+ fn extract_text(v: &serde_json::Value) -> String {+ v.pointer("/result/content/0/text").and_then(|t| t.as_str())+ .or_else(|| v.pointer("/error").and_then(|e| e.as_str()))+ .map(|s| s.to_owned()).unwrap_or_default()+ }++ let lw = req(&mut pipe, r#"{"method":"call","name":"list_windows","args":{}}"#);+ let lwv: serde_json::Value = serde_json::from_str(lw.trim()).unwrap();+ let wins = lwv.pointer("/result/structuredContent/windows").and_then(|w| w.as_array()).unwrap();+ let target = wins.iter().find(|w| {+ let n = w.pointer("/app_name").and_then(|s| s.as_str()).unwrap_or("").to_lowercase();+ n.contains("chrome")+ }).expect("no chrome");+ let pid = target.pointer("/pid").and_then(|n| n.as_u64()).unwrap();+ let wid = target.pointer("/window_id").and_then(|n| n.as_u64()).unwrap();++ // 1. Invalid region (x2 ≤ x1).+ let r1 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"zoom","args":{{"pid":{pid},"window_id":{wid},"x1":100,"y1":100,"x2":50,"y2":200}}}}"#));+ let e1 = extract_text(&serde_json::from_str(r1.trim()).unwrap());+ assert!(e1.contains("Invalid region: x2 must be > x1 and y2 must be > y1"),+ "Invalid-region wording wrong: {e1:?}");+ println!("Invalid-region err OK");++ // 2. Region too wide.+ let r2 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"zoom","args":{{"pid":{pid},"window_id":{wid},"x1":0,"y1":0,"x2":600,"y2":100}}}}"#));+ let e2 = extract_text(&serde_json::from_str(r2.trim()).unwrap());+ assert!(e2.contains("Zoom region too wide") && e2.contains("> 500 px max"),+ "Too-wide wording wrong: {e2:?}");+ println!("Too-wide err OK");++ // 3. Real zoom — small region.+ let r3 = req(&mut pipe, &format!(+ r#"{{"method":"call","name":"zoom","args":{{"pid":{pid},"window_id":{wid},"x1":50,"y1":50,"x2":150,"y2":150}}}}"#));+ let v3: serde_json::Value = serde_json::from_str(r3.trim()).unwrap();+ let content = v3.pointer("/result/content").and_then(|c| c.as_array()).unwrap();+ let img_present = content.iter().any(|c| c.get("type").and_then(|t| t.as_str()) == Some("image"));+ let text = content.iter().find_map(|c| {+ if c.get("type").and_then(|t| t.as_str()) == Some("text") {+ c.get("text").and_then(|t| t.as_str())+ } else { None }+ }).unwrap_or("");+ assert!(img_present, "zoom response should include image content");+ assert!(text.starts_with("✅ Zoomed region captured at native resolution.")+ && text.contains("from_zoom=true"),+ "zoom text doesn't match Swift format: {text:?}");+ println!("Real zoom OK");++ println!("\n✅ PASS: zoom error wording + Swift text format verified");+}++#[cfg(not(target_os = "windows"))]+fn main() {}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs b/libs/cua-driver-rs/crates/platform-windows/src/capture.rsnew file mode 100644index 0000000000..a6c920c896--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs@@ -0,0 +1,333 @@+//! Window screenshot via PrintWindow + GDI BitBlt on Windows.+//!+//! PW_RENDERFULLCONTENT (0x2) renders the window contents even if it is+//! occluded or off-screen. The result is encoded as base64 PNG in memory.++use anyhow::{bail, Result};+use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};+use windows::Win32::Foundation::HWND;+use windows::Win32::Graphics::Gdi::{+ BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC,+ GetDIBits, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS,+ RGBQUAD, SRCCOPY,+};+use windows::Win32::Graphics::Gdi::{GetWindowDC, ReleaseDC};+use windows::Win32::Storage::Xps::{PrintWindow, PRINT_WINDOW_FLAGS};+const PW_RENDERFULLCONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2u32);++/// Capture a window by HWND, returning raw PNG bytes.+pub fn screenshot_window_bytes(hwnd: u64) -> Result<Vec<u8>> {+ unsafe { screenshot_window_bytes_unsafe(hwnd) }+}++/// Capture a window by HWND, returning (base64_png, width, height).+pub fn screenshot_window(hwnd: u64) -> Result<(String, u32, u32)> {+ let png_bytes = screenshot_window_bytes(hwnd)?;+ let (w, h) = {+ if png_bytes.len() < 24 { bail!("PNG too small"); }+ let w = u32::from_be_bytes([png_bytes[16], png_bytes[17], png_bytes[18], png_bytes[19]]);+ let h = u32::from_be_bytes([png_bytes[20], png_bytes[21], png_bytes[22], png_bytes[23]]);+ (w, h)+ };+ Ok((BASE64.encode(&png_bytes), w, h))+}++unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result<Vec<u8>> {+ use windows::Win32::UI::WindowsAndMessaging::GetClientRect;+ use windows::Win32::Foundation::RECT;++ let hwnd = HWND(hwnd as *mut _);++ let mut rect = RECT::default();+ GetClientRect(hwnd, &mut rect)?;+ let w = (rect.right - rect.left) as i32;+ let h = (rect.bottom - rect.top) as i32;+ if w <= 0 || h <= 0 {+ bail!("Window has zero/negative client size: {}x{}", w, h);+ }++ let screen_dc = GetWindowDC(hwnd);+ let mem_dc = CreateCompatibleDC(screen_dc);+ let bitmap = CreateCompatibleBitmap(screen_dc, w, h);+ let old_bitmap = SelectObject(mem_dc, bitmap);++ let pw_ok = PrintWindow(hwnd, mem_dc, PW_RENDERFULLCONTENT);+ if !pw_ok.as_bool() {+ BitBlt(mem_dc, 0, 0, w, h, screen_dc, 0, 0, SRCCOPY)?;+ }++ let mut bmi = BITMAPINFO {+ bmiHeader: BITMAPINFOHEADER {+ biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,+ biWidth: w,+ biHeight: -h,+ biPlanes: 1,+ biBitCount: 32,+ biCompression: BI_RGB.0,+ biSizeImage: (w * h * 4) as u32,+ ..Default::default()+ },+ bmiColors: [RGBQUAD::default(); 1],+ };++ let pixel_count = (w * h) as usize;+ let mut pixels = vec![0u8; pixel_count * 4];+ let ok = GetDIBits(mem_dc, bitmap, 0, h as u32, Some(pixels.as_mut_ptr() as *mut _), &mut bmi, DIB_RGB_COLORS);++ SelectObject(mem_dc, old_bitmap);+ DeleteObject(bitmap);+ DeleteDC(mem_dc);+ ReleaseDC(hwnd, screen_dc);++ if ok == 0 { bail!("GetDIBits returned 0"); }++ encode_bgra_to_png(&pixels, w as u32, h as u32)+}+++/// Minimal BGRA→PNG encoder (no external dependency on image/lodepng).+fn encode_bgra_to_png(bgra: &[u8], w: u32, h: u32) -> Result<Vec<u8>> {+ // Convert BGRA to RGBA.+ let mut rgba = bgra.to_vec();+ for px in rgba.chunks_exact_mut(4) {+ px.swap(0, 2); // B↔R+ }++ // Build PNG in-memory using the flate2 + adler32 approach manually,+ // or simply call out to a temp file via GDI+ / stb_image_write.+ // For simplicity, use the `image` crate if available, otherwise write raw+ // uncompressed PNG (which most tools accept).+ write_uncompressed_png(&rgba, w, h)+}++/// Write a minimal uncompressed PNG (IDAT with zlib level 0 = store).+fn write_uncompressed_png(rgba: &[u8], w: u32, h: u32) -> Result<Vec<u8>> {+ let mut out = Vec::with_capacity(rgba.len() + 4096);++ // PNG signature.+ out.extend_from_slice(b"\x89PNG\r\n\x1a\n");++ // IHDR.+ let mut ihdr = Vec::with_capacity(13);+ ihdr.extend_from_slice(&w.to_be_bytes());+ ihdr.extend_from_slice(&h.to_be_bytes());+ ihdr.push(8); // bit depth+ ihdr.push(2); // color type: RGB (we'll drop alpha for simplicity) — actually use 6 for RGBA+ ihdr[9] = 6; // RGBA+ ihdr.push(0); // compression+ ihdr.push(0); // filter+ ihdr.push(0); // interlace+ // Rewrite properly.+ let ihdr: [u8; 13] = [+ (w >> 24) as u8, (w >> 16) as u8, (w >> 8) as u8, w as u8,+ (h >> 24) as u8, (h >> 16) as u8, (h >> 8) as u8, h as u8,+ 8, // bit depth+ 6, // RGBA+ 0, // deflate+ 0, // adaptive filter+ 0, // no interlace+ ];+ write_png_chunk(&mut out, b"IHDR", &ihdr);++ // IDAT: zlib-wrap with store (DEFLATE BTYPE=00).+ // Build raw scanlines: [filter_byte(0), row_pixels...]+ let row_bytes = (w * 4) as usize;+ let mut raw = Vec::with_capacity((row_bytes + 1) * h as usize);+ for row in 0..h as usize {+ raw.push(0u8); // filter = None+ raw.extend_from_slice(&rgba[row * row_bytes..(row + 1) * row_bytes]);+ }+ let zlib_data = zlib_store(&raw);+ write_png_chunk(&mut out, b"IDAT", &zlib_data);++ // IEND.+ write_png_chunk(&mut out, b"IEND", &[]);++ Ok(out)+}++fn write_png_chunk(out: &mut Vec<u8>, name: &[u8; 4], data: &[u8]) {+ let len = data.len() as u32;+ out.extend_from_slice(&len.to_be_bytes());+ out.extend_from_slice(name);+ out.extend_from_slice(data);+ let crc = crc32_ieee(name, data);+ out.extend_from_slice(&crc.to_be_bytes());+}++/// zlib store wrapper (BTYPE=00 non-compressed blocks, max 65535 bytes/block).+fn zlib_store(data: &[u8]) -> Vec<u8> {+ let adler = adler32(data);+ let mut out = Vec::new();+ // zlib header: CMF=0x78, FLG=0x01 (no dict, check bits).+ out.push(0x78);+ out.push(0x01);+ // DEFLATE non-compressed blocks.+ let mut pos = 0;+ while pos < data.len() || data.is_empty() {+ let end = (pos + 65535).min(data.len());+ let is_last = end == data.len();+ let blen = (end - pos) as u16;+ out.push(if is_last { 1 } else { 0 }); // BFINAL | BTYPE=00+ out.extend_from_slice(&blen.to_le_bytes());+ out.extend_from_slice(&(!blen).to_le_bytes());+ out.extend_from_slice(&data[pos..end]);+ pos = end;+ if data.is_empty() { break; }+ }+ // Adler-32 checksum (big-endian).+ out.extend_from_slice(&adler.to_be_bytes());+ out+}++fn adler32(data: &[u8]) -> u32 {+ let mut s1: u32 = 1;+ let mut s2: u32 = 0;+ for &b in data {+ s1 = (s1 + b as u32) % 65521;+ s2 = (s2 + s1) % 65521;+ }+ (s2 << 16) | s1+}++/// Capture the primary display (full screen), returning raw PNG bytes.+pub fn screenshot_display_bytes() -> Result<Vec<u8>> {+ unsafe {+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};+ let w = GetSystemMetrics(SM_CXSCREEN);+ let h = GetSystemMetrics(SM_CYSCREEN);+ if w <= 0 || h <= 0 { bail!("Could not get screen metrics"); }+ let screen_dc = GetDC(HWND::default());+ let mem_dc = CreateCompatibleDC(screen_dc);+ let bitmap = CreateCompatibleBitmap(screen_dc, w, h);+ let old_bitmap = SelectObject(mem_dc, bitmap);+ BitBlt(mem_dc, 0, 0, w, h, screen_dc, 0, 0, SRCCOPY)?;+ let mut bmi = BITMAPINFO {+ bmiHeader: BITMAPINFOHEADER {+ biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,+ biWidth: w, biHeight: -h, biPlanes: 1, biBitCount: 32,+ biCompression: BI_RGB.0,+ biSizeImage: (w * h * 4) as u32, ..Default::default()+ },+ bmiColors: [RGBQUAD::default(); 1],+ };+ let mut pixels = vec![0u8; (w * h * 4) as usize];+ let ok = GetDIBits(mem_dc, bitmap, 0, h as u32, Some(pixels.as_mut_ptr() as *mut _), &mut bmi, DIB_RGB_COLORS);+ SelectObject(mem_dc, old_bitmap);+ DeleteObject(bitmap);+ DeleteDC(mem_dc);+ ReleaseDC(HWND::default(), screen_dc);+ if ok == 0 { bail!("GetDIBits returned 0"); }+ encode_bgra_to_png(&pixels, w as u32, h as u32)+ }+}++/// Capture primary display, returning (base64_png, width, height).+pub fn screenshot_display() -> Result<(String, u32, u32)> {+ let png_bytes = screenshot_display_bytes()?;+ if png_bytes.len() < 24 { bail!("PNG too small"); }+ let w = u32::from_be_bytes([png_bytes[16], png_bytes[17], png_bytes[18], png_bytes[19]]);+ let h = u32::from_be_bytes([png_bytes[20], png_bytes[21], png_bytes[22], png_bytes[23]]);+ Ok((BASE64.encode(&png_bytes), w, h))+}++/// Convert PNG bytes to JPEG at the given quality (1–95).+pub fn png_bytes_to_jpeg(png_bytes: &[u8], quality: u8) -> Result<Vec<u8>> {+ let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?;+ let mut buf = Vec::new();+ {+ let mut cursor = std::io::Cursor::new(&mut buf);+ let encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut cursor, quality);+ img.write_with_encoder(encoder)?;+ }+ Ok(buf)+}++/// Downscale `png_bytes` so neither dimension exceeds `max_dim`.+/// If `max_dim == 0` or the image already fits, returns a copy of the original bytes unchanged.+pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> Result<Vec<u8>> {+ if max_dim == 0 {+ return Ok(png_bytes.to_vec());+ }+ if png_bytes.len() < 24 { bail!("PNG too small"); }+ let w = u32::from_be_bytes([png_bytes[16], png_bytes[17], png_bytes[18], png_bytes[19]]);+ let h = u32::from_be_bytes([png_bytes[20], png_bytes[21], png_bytes[22], png_bytes[23]]);+ if w <= max_dim && h <= max_dim {+ return Ok(png_bytes.to_vec());+ }+ let scale = max_dim as f64 / w.max(h) as f64;+ let new_w = (w as f64 * scale).round() as u32;+ let new_h = (h as f64 * scale).round() as u32;+ let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?;+ let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3);+ let mut out = Vec::new();+ resized.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?;+ Ok(out)+}++/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return modified PNG bytes.+/// Used by recording's click-marker callback to produce click.png.+pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result<Vec<u8>> {+ let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?;+ let (w, h) = (img.width(), img.height());+ let mut img = img.to_rgba8();++ let arm_len = (w as f64 / 40.0).max(12.0) as i32;+ let line_w = ((w as f64 / 400.0).max(1.5)) as i32;+ let red = image::Rgba([255u8, 26, 26, 242]);+ let cx = cx as i32;+ let cy = cy as i32;++ for lw in 0..=line_w {+ let off = lw - line_w / 2;+ for dx in -arm_len..=arm_len {+ if let Some(p) = img.get_pixel_mut_checked(+ (cx + dx).clamp(0, w as i32 - 1) as u32,+ (cy + off).clamp(0, h as i32 - 1) as u32,+ ) { *p = red; }+ }+ for dy in -arm_len..=arm_len {+ if let Some(p) = img.get_pixel_mut_checked(+ (cx + off).clamp(0, w as i32 - 1) as u32,+ (cy + dy).clamp(0, h as i32 - 1) as u32,+ ) { *p = red; }+ }+ }++ let mut out = Vec::new();+ image::DynamicImage::ImageRgba8(img)+ .write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?;+ Ok(out)+}++/// Parse width and height from a PNG IHDR chunk.+pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> {+ if data.len() < 24 { bail!("PNG data too small"); }+ if &data[0..8] != b"\x89PNG\r\n\x1a\n" { bail!("Not a PNG"); }+ let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]);+ let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]);+ Ok((w, h))+}++fn crc32_ieee(name: &[u8], data: &[u8]) -> u32 {+ const TABLE: [u32; 256] = {+ let mut t = [0u32; 256];+ let mut i = 0usize;+ while i < 256 {+ let mut c = i as u32;+ let mut j = 0;+ while j < 8 {+ c = if c & 1 != 0 { 0xEDB88320 ^ (c >> 1) } else { c >> 1 };+ j += 1;+ }+ t[i] = c;+ i += 1;+ }+ t+ };+ let mut crc: u32 = !0u32;+ for &b in name.iter().chain(data.iter()) {+ crc = TABLE[((crc ^ b as u32) & 0xFF) as usize] ^ (crc >> 8);+ }+ !crc+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rsnew file mode 100644index 0000000000..308fa7427a--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs@@ -0,0 +1,157 @@+//! Background keyboard injection via PostMessage.+//!+//! WM_CHAR — posts a character code; simpler and more reliable for text entry.+//! WM_KEYDOWN/WM_KEYUP — used for non-printable keys (Enter, Tab, arrows, F-keys, etc.)+//!+//! All messages are posted async (PostMessageW), so they do NOT steal focus.++use anyhow::{bail, Result};+use std::thread::sleep;+use std::time::Duration;+use windows::Win32::Foundation::{HWND, LPARAM, WPARAM};+use windows::Win32::UI::Input::KeyboardAndMouse::{+ MapVirtualKeyW, MAPVK_VK_TO_VSC, VIRTUAL_KEY,+};+use windows::Win32::UI::WindowsAndMessaging::{+ PostMessageW, WM_CHAR, WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, WM_SYSKEYUP,+};++const KEY_DELAY_MS: u64 = 4;++/// Post a Unicode character as WM_CHAR.+pub fn post_char(hwnd: u64, ch: char) -> Result<()> {+ let hwnd = HWND(hwnd as *mut _);+ let code = ch as u32 as usize;+ unsafe {+ PostMessageW(hwnd, WM_CHAR, WPARAM(code), LPARAM(1))?;+ }+ Ok(())+}++/// Post all characters in a string as WM_CHAR messages with inter-key delay.+pub fn post_type_text(hwnd: u64, text: &str) -> Result<()> {+ for ch in text.chars() {+ post_char(hwnd, ch)?;+ sleep(Duration::from_millis(KEY_DELAY_MS));+ }+ Ok(())+}++/// Post all characters in `text` as WM_CHAR messages with a configurable+/// inter-character delay (on top of the baseline KEY_DELAY_MS gap).+pub fn post_type_text_with_delay(hwnd: u64, text: &str, inter_char_ms: u64) -> Result<()> {+ for ch in text.chars() {+ post_char(hwnd, ch)?;+ sleep(Duration::from_millis(KEY_DELAY_MS + inter_char_ms));+ }+ Ok(())+}++/// Press a named key (and optional modifiers) via WM_KEYDOWN/WM_KEYUP.+pub fn post_key(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {+ let hwnd_win = HWND(hwnd as *mut _);+ let vk = key_name_to_vk(key)?;+ let has_alt = modifiers.iter().any(|m| *m == "alt" || *m == "menu");++ let scan = unsafe { MapVirtualKeyW(vk.0 as u32, MAPVK_VK_TO_VSC) };+ let repeat_lp = |scan: u32, extended: bool, key_up: bool| {+ let mut lp: u32 = 1; // repeat count+ lp |= scan << 16;+ if extended { lp |= 1 << 24; }+ if key_up { lp |= (1 << 30) | (1 << 31); }+ LPARAM(lp as isize)+ };++ let (down_msg, up_msg) = if has_alt {+ (WM_SYSKEYDOWN, WM_SYSKEYUP)+ } else {+ (WM_KEYDOWN, WM_KEYUP)+ };++ let mod_vks: Vec<VIRTUAL_KEY> = modifiers.iter()+ .filter_map(|m| modifier_vk(m))+ .collect();++ unsafe {+ // Press modifiers.+ for mvk in &mod_vks {+ let ms = MapVirtualKeyW(mvk.0 as u32, MAPVK_VK_TO_VSC);+ PostMessageW(hwnd_win, down_msg, WPARAM(mvk.0 as usize), repeat_lp(ms, false, false))?;+ }+ // Press key.+ PostMessageW(hwnd_win, down_msg, WPARAM(vk.0 as usize), repeat_lp(scan, is_extended(vk), false))?;+ sleep(Duration::from_millis(KEY_DELAY_MS));+ // Release key.+ PostMessageW(hwnd_win, up_msg, WPARAM(vk.0 as usize), repeat_lp(scan, is_extended(vk), true))?;+ // Release modifiers (reverse order).+ for mvk in mod_vks.iter().rev() {+ let ms = MapVirtualKeyW(mvk.0 as u32, MAPVK_VK_TO_VSC);+ PostMessageW(hwnd_win, up_msg, WPARAM(mvk.0 as usize), repeat_lp(ms, false, true))?;+ }+ }+ Ok(())+}++fn modifier_vk(name: &str) -> Option<VIRTUAL_KEY> {+ use windows::Win32::UI::Input::KeyboardAndMouse::*;+ match name.to_lowercase().as_str() {+ "ctrl" | "control" => Some(VK_CONTROL),+ "shift" => Some(VK_SHIFT),+ "alt" | "menu" => Some(VK_MENU),+ "win" | "meta" | "windows" => Some(VK_LWIN),+ _ => None,+ }+}++fn is_extended(vk: VIRTUAL_KEY) -> bool {+ use windows::Win32::UI::Input::KeyboardAndMouse::*;+ matches!(vk,+ VK_DELETE | VK_INSERT | VK_HOME | VK_END | VK_PRIOR | VK_NEXT |+ VK_UP | VK_DOWN | VK_LEFT | VK_RIGHT |+ VK_RCONTROL | VK_RMENU | VK_RWIN | VK_NUMLOCK | VK_SNAPSHOT+ )+}++fn key_name_to_vk(key: &str) -> Result<VIRTUAL_KEY> {+ use windows::Win32::UI::Input::KeyboardAndMouse::*;+ let vk = match key.to_lowercase().as_str() {+ "enter" | "return" => VK_RETURN,+ "tab" => VK_TAB,+ "escape" | "esc" => VK_ESCAPE,+ "space" | " " => VK_SPACE,+ "backspace" => VK_BACK,+ "delete" | "del" => VK_DELETE,+ "insert" | "ins" => VK_INSERT,+ "home" => VK_HOME,+ "end" => VK_END,+ "pageup" | "pgup" => VK_PRIOR,+ "pagedown" | "pgdn" => VK_NEXT,+ "up" => VK_UP,+ "down" => VK_DOWN,+ "left" => VK_LEFT,+ "right" => VK_RIGHT,+ "f1" => VK_F1, "f2" => VK_F2, "f3" => VK_F3, "f4" => VK_F4,+ "f5" => VK_F5, "f6" => VK_F6, "f7" => VK_F7, "f8" => VK_F8,+ "f9" => VK_F9, "f10" => VK_F10, "f11" => VK_F11, "f12" => VK_F12,+ "ctrl" | "control" => VK_CONTROL,+ "shift" => VK_SHIFT,+ "alt" => VK_MENU,+ "win" | "windows" | "meta" | "command" | "cmd" => VK_LWIN,+ "capslock" => VK_CAPITAL,+ "numlock" => VK_NUMLOCK,+ _ => {+ // Single printable character.+ let ch = key.chars().next()+ .ok_or_else(|| anyhow::anyhow!("Empty key name"))?;+ // VkKeyScanW returns VK in low byte.+ let vk_scan = unsafe {+ windows::Win32::UI::Input::KeyboardAndMouse::VkKeyScanW(ch as u16)+ };+ if vk_scan == -1i16 as u16 as i16 {+ bail!("Unknown key: {key}");+ }+ VIRTUAL_KEY((vk_scan & 0xFF) as u16)+ }+ };+ Ok(vk)+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rsnew file mode 100644index 0000000000..1244aa4771--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs@@ -0,0 +1,16 @@+//! Background input synthesis for Windows.+//!+//! Strategy (matches CuaDriver.Win reference):+//! - Mouse: PostMessageW(WM_LBUTTONDOWN/UP) with packed LPARAM(x,y).+//! PostMessage is async and does NOT activate the target window.+//! - Keyboard text: PostMessageW(WM_CHAR) per character.+//! - Key press: PostMessageW(WM_KEYDOWN) + PostMessageW(WM_KEYUP).+//!+//! For clicks, coordinates are window-client-relative (ScreenToClient applied+//! if screen coords are given). We pack with MAKELPARAM(x,y).++pub mod mouse;+pub mod keyboard;++pub use mouse::{post_click, post_click_screen};+pub use keyboard::{post_char, post_key, post_type_text, post_type_text_with_delay};diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rsnew file mode 100644index 0000000000..ebc7079da1--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs@@ -0,0 +1,158 @@+//! Background mouse injection via PostMessage.+//!+//! All clicks target the **deepest child** HWND at the click point+//! (via ChildWindowFromPointEx), so the message never reaches the top-level+//! chrome that would call SetForegroundWindow in response to WM_LBUTTONDOWN.++use anyhow::Result;+use std::thread::sleep;+use std::time::Duration;+use windows::Win32::Foundation::{HWND, LPARAM, POINT, WPARAM};+use windows::Win32::Graphics::Gdi::{ClientToScreen, ScreenToClient};+use windows::Win32::UI::WindowsAndMessaging::{+ ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT,+ PostMessageW, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP,+ WM_MOUSEMOVE, WM_RBUTTONDOWN, WM_RBUTTONUP,+};++const MK_LBUTTON: u32 = 0x0001;+const MK_MBUTTON: u32 = 0x0010;+const MK_RBUTTON: u32 = 0x0002;++const CLICK_DELAY_MS: u64 = 35;++/// Walk from `root` down to the deepest visible child that contains+/// `screen_pt`, mirroring trope-cua's DeepestChildFromScreenPoint.+///+/// Posting to the deepest child avoids the top-level window responding to+/// WM_LBUTTONDOWN by activating itself (focus-steal).+fn deepest_child(root: HWND, screen_pt: POINT) -> (HWND, POINT) {+ let mut current = root;+ for _ in 0..16 {+ let mut client = screen_pt;+ unsafe { let _ = ScreenToClient(current, &mut client); }+ let child = unsafe {+ ChildWindowFromPointEx(+ current, client,+ CWP_SKIPINVISIBLE | CWP_SKIPDISABLED | CWP_SKIPTRANSPARENT,+ )+ };+ // No deeper child, or same window, or outside root's subtree.+ if child.is_invalid() || child == current {+ break;+ }+ // Verify the child is actually a descendant of root.+ let is_child = unsafe { windows::Win32::UI::WindowsAndMessaging::IsChild(root, child) };+ if !is_child.as_bool() && child != root {+ break;+ }+ current = child;+ }+ // Return child-local client coordinates for `current`.+ let mut client = screen_pt;+ unsafe { let _ = ScreenToClient(current, &mut client); }+ (current, client)+}++/// Post a click at **client-area** coordinates of `root_hwnd`.+///+/// Resolves to the deepest child HWND at the click point before posting,+/// so top-level browser/chrome windows don't activate in response.+pub fn post_click(root: u64, x: i32, y: i32, count: usize, button: &str) -> Result<()> {+ let root_hwnd = HWND(root as *mut _);++ // Convert root-local client → screen.+ let mut screen_pt = POINT { x, y };+ unsafe { let _ = ClientToScreen(root_hwnd, &mut screen_pt); }++ // Find deepest child and its local client coordinates.+ let (target, client) = deepest_child(root_hwnd, screen_pt);++ post_click_on(target, client.x, client.y, count, button)+}++/// Post a click at **screen** coordinates, resolving the deepest child of+/// `root_hwnd` at that point. Call this when you already have screen coords.+pub fn post_click_screen(root: u64, sx: i32, sy: i32, count: usize, button: &str) -> Result<()> {+ let root_hwnd = HWND(root as *mut _);+ let screen_pt = POINT { x: sx, y: sy };+ let (target, client) = deepest_child(root_hwnd, screen_pt);+ post_click_on(target, client.x, client.y, count, button)+}++/// Internal: post click messages to `hwnd` using its own client coordinates.+fn post_click_on(hwnd: HWND, x: i32, y: i32, count: usize, button: &str) -> Result<()> {+ let (down_msg, up_msg, mk_flag) = match button {+ "right" => (WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON),+ "middle" => (WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON),+ _ => (WM_LBUTTONDOWN, WM_LBUTTONUP, MK_LBUTTON),+ };+ let lparam = make_lparam(x, y);+ let wdown = WPARAM(mk_flag as usize);+ let wup = WPARAM(0);++ for i in 0..count {+ unsafe {+ // WM_MOUSEMOVE first so hover state is correct before the click.+ PostMessageW(hwnd, WM_MOUSEMOVE, WPARAM(0), lparam)?;+ PostMessageW(hwnd, down_msg, wdown, lparam)?;+ sleep(Duration::from_millis(CLICK_DELAY_MS));+ PostMessageW(hwnd, up_msg, wup, lparam)?;+ }+ if i + 1 < count {+ sleep(Duration::from_millis(80));+ }+ }+ Ok(())+}++/// Post a press-drag-release gesture via PostMessage.+///+/// Coordinates are root-hwnd client-area relative.+pub fn post_drag(+ hwnd: u64,+ from_x: i32,+ from_y: i32,+ to_x: i32,+ to_y: i32,+ duration_ms: u64,+ steps: usize,+ button: &str,+) -> Result<()> {+ let root = HWND(hwnd as *mut _);+ let (down_msg, up_msg, mk_flag) = match button {+ "right" => (WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON),+ "middle" => (WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON),+ _ => (WM_LBUTTONDOWN, WM_LBUTTONUP, MK_LBUTTON),+ };+ let wparam = WPARAM(mk_flag as usize);+ let steps = steps.max(1);+ let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };++ unsafe {+ PostMessageW(root, down_msg, wparam, make_lparam(from_x, from_y))?;+ }+ sleep(Duration::from_millis(CLICK_DELAY_MS));++ for i in 1..=steps {+ let t = i as f64 / steps as f64;+ let ix = from_x + ((to_x - from_x) as f64 * t).round() as i32;+ let iy = from_y + ((to_y - from_y) as f64 * t).round() as i32;+ unsafe {+ PostMessageW(root, WM_MOUSEMOVE, wparam, make_lparam(ix, iy))?;+ }+ if step_delay_ms > 0 {+ sleep(Duration::from_millis(step_delay_ms));+ }+ }++ unsafe {+ PostMessageW(root, up_msg, WPARAM(0), make_lparam(to_x, to_y))?;+ }+ Ok(())+}++/// Pack two 16-bit integers into a LPARAM (low word = x, high word = y).+fn make_lparam(x: i32, y: i32) -> LPARAM {+ LPARAM((((y as u16 as u32) << 16) | (x as u16 as u32)) as isize)+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/lib.rs b/libs/cua-driver-rs/crates/platform-windows/src/lib.rsnew file mode 100644index 0000000000..d119217bdc--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/lib.rs@@ -0,0 +1,39 @@+//! Windows platform backend for cua-driver-rs.+//!+//! Background automation on Windows via:+//! - UI Automation (UIA / MSAA) for accessibility tree walking+//! - PostMessage(WM_LBUTTONDOWN/WM_LBUTTONUP) for background mouse injection+//! - PostMessage(WM_CHAR / WM_KEYDOWN/UP) for keyboard events+//! - Win32 EnumWindows / CreateToolhelp32Snapshot for enumeration+//! - PrintWindow / GDI BitBlt for screenshots+//!+//! Reference: /tmp/trope-cua/src/CuaDriver.Win/++use mcp_server::tool::ToolRegistry;++pub mod tools;+pub mod overlay;++#[cfg(target_os = "windows")]+pub mod win32;++#[cfg(target_os = "windows")]+pub mod uia;++#[cfg(target_os = "windows")]+pub mod input;++#[cfg(target_os = "windows")]+pub mod capture;++pub fn register_tools() -> ToolRegistry {+ tools::build_registry()+}++pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig) -> ToolRegistry {+ if cfg.enabled {+ overlay::init(cfg.clone());+ overlay::run_on_thread();+ }+ tools::build_registry()+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/overlay.rs b/libs/cua-driver-rs/crates/platform-windows/src/overlay.rsnew file mode 100644index 0000000000..3d0941e21b--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/overlay.rs@@ -0,0 +1,768 @@+//! Win32 agent-cursor overlay — transparent, click-through layered window.+//!+//! Matches the C# reference in CuaDriver.Win/Cursor/AgentCursorOverlay.cs:+//!+//! - Extended style: `WS_EX_TRANSPARENT | WS_EX_LAYERED | WS_EX_NOACTIVATE | WS_EX_TOOLWINDOW`+//! - Spans the virtual screen (all monitors).+//! - Render loop: dedicated STA thread, ~125 Hz (8ms timer via `SetTimer`).+//! - Pixel pipeline: `tiny-skia` → BGRA DIB → `UpdateLayeredWindow` per-pixel alpha.+//! - Z-ordering: every 80ms call `SetWindowPos` to stay just above the pinned target.+//! - Idle-hide: fade out over 180ms once `idle_hide_ms` has elapsed with no activity.++#![allow(non_snake_case, non_upper_case_globals)]++use std::sync::{Mutex, OnceLock};+use std::time::{Duration, Instant};++use cursor_overlay::{+ CursorConfig, CursorShape, MotionConfig, OverlayCommand, Palette, PathPlanner, PathState,+ PlannedPath,+};++// ── Global channel ────────────────────────────────────────────────────────++static CMD_TX: OnceLock<std::sync::mpsc::SyncSender<OverlayCommand>> = OnceLock::new();+static CMD_RX_CELL: Mutex<Option<std::sync::mpsc::Receiver<OverlayCommand>>> = Mutex::new(None);+static RENDER: Mutex<Option<RenderState>> = Mutex::new(None);++pub fn init(cfg: CursorConfig) {+ let (tx, rx) = std::sync::mpsc::sync_channel(4096);+ let _ = CMD_TX.set(tx);+ *CMD_RX_CELL.lock().unwrap() = Some(rx);+ *RENDER.lock().unwrap() = Some(RenderState::new(cfg));+}++pub fn send_command(cmd: OverlayCommand) {+ if let Some(tx) = CMD_TX.get() {+ let _ = tx.try_send(cmd);+ }+}++/// Returns the current glide duration in milliseconds (default 750).+/// Used by the click path to wait for the animation before firing ClickPulse.+pub fn glide_duration_ms() -> f64 {+ RENDER.lock().ok()+ .and_then(|g| g.as_ref().map(|rs| rs.motion.glide_duration_ms))+ .unwrap_or(750.0)+}++/// Returns true if the cursor overlay is currently enabled/visible.+pub fn is_enabled() -> bool {+ RENDER.lock().ok()+ .and_then(|g| g.as_ref().map(|rs| rs.visible))+ .unwrap_or(false)+}++/// Snapshot the current motion config (start_handle / end_handle / arc_size /+/// arc_flow / spring / glide_duration_ms / dwell_after_click_ms /+/// idle_hide_ms). Mirrors macOS `current_motion()` so+/// `get_agent_cursor_state` can report the live values.+pub fn current_motion() -> MotionConfig {+ RENDER.lock().ok()+ .and_then(|g| g.as_ref().map(|rs| rs.motion.clone()))+ .unwrap_or_default()+}++/// Returns the current cursor position in screen coordinates.+pub fn current_position() -> (f64, f64) {+ RENDER.lock().ok()+ .and_then(|g| g.as_ref().map(|rs| rs.pos))+ .unwrap_or((-200.0, -200.0))+}++/// Returns true if the cursor is still at the off-screen initial position+/// (-200, -200), meaning it has never been positioned on screen yet.+pub fn is_at_initial_position() -> bool {+ RENDER.lock().ok()+ .and_then(|g| g.as_ref().map(|rs| rs.pos.0 < 0.0 && rs.pos.1 < 0.0))+ .unwrap_or(true)+}++/// Spin up the overlay on a dedicated thread (STA for Win32 message loop).+/// This is a non-blocking call — the overlay runs on its own thread.+pub fn run_on_thread() {+ let rx = match CMD_RX_CELL.lock().unwrap().take() {+ Some(r) => r,+ None => return, // init() not called; overlay disabled+ };++ let cfg = {+ let guard = RENDER.lock().unwrap();+ match &*guard {+ Some(rs) => rs.cfg.clone(),+ None => return,+ }+ };++ if !cfg.enabled {+ return;+ }++ std::thread::Builder::new()+ .name("cua-overlay-win".into())+ .spawn(move || {+ // Windows message loops must run on the same thread that created the window.+ run_overlay_thread(cfg, rx);+ })+ .expect("spawn overlay thread");+}++// ── Animation state (same structure as macOS) ─────────────────────────────++struct RenderState {+ cfg: CursorConfig,+ palette: Palette,+ motion: MotionConfig,+ pos: (f64, f64),+ heading: f64,+ path: Option<PlannedPath>,+ dist: f64,+ start_t: Instant,+ spring: Option<Spring>,+ spring_tgt: Option<(f64, f64, f64)>,+ click_t: Option<f64>,+ shape: Option<CursorShape>,+ visible: bool,+ idle_secs: f64,+ idle_alpha: f64,+ pinned_wid: Option<u64>,+ gradient_colors: Vec<[u8; 4]>,+ bloom_override: Option<[u8; 4]>,+ last_tick: Instant,+ // Virtual screen dimensions set after window creation.+ virt_x: i32,+ virt_y: i32,+ virt_w: i32,+ virt_h: i32,+}++#[derive(Clone, Copy)]+struct Spring { ox: f64, oy: f64, vx: f64, vy: f64 }++impl RenderState {+ fn new(cfg: CursorConfig) -> Self {+ let palette = cfg.palette();+ let motion = cfg.motion.clone();+ let shape = cfg.shape.clone();+ RenderState {+ cfg, palette, motion, shape,+ pos: (-200.0, -200.0),+ heading: std::f64::consts::FRAC_PI_4,+ path: None,+ dist: 0.0,+ start_t: Instant::now(),+ spring: None,+ spring_tgt: None,+ click_t: None,+ visible: true,+ idle_secs: 0.0,+ idle_alpha: 1.0,+ pinned_wid: None,+ gradient_colors: vec![],+ bloom_override: None,+ last_tick: Instant::now(),+ virt_x: 0, virt_y: 0, virt_w: 1920, virt_h: 1080,+ }+ }++ fn tick(&mut self, dt: f64) {+ let spring_k = self.motion.spring * 400.0;+ let spring_c = self.motion.spring * 20.0;++ if let Some(ref p) = self.path {+ // Speed-based motion: 16*u²*(1-u)² peaks at exactly 1.0 at u=0.5,+ // matching Swift AgentCursorRenderer (peakSpeed=900, min=300/200).+ let path_frac = (self.dist / p.length.max(1.0)).clamp(0.0, 1.0);+ let profile = 16.0 * path_frac * path_frac * (1.0 - path_frac) * (1.0 - path_frac);+ let floor = if path_frac < 0.5 { self.motion.min_start_speed } else { self.motion.min_end_speed };+ let speed = (floor + (self.motion.peak_speed - floor) * profile).max(floor);+ self.dist += speed * dt;++ let path_len = p.length.max(1.0);+ if self.dist >= path_len {+ let end = p.sample(path_len);+ let end_heading = p.end_visual_heading;+ let vh = end.heading;+ self.spring = Some(Spring {+ ox: 0.0, oy: 0.0,+ vx: speed * 0.5 * vh.cos(),+ vy: speed * 0.5 * vh.sin(),+ });+ self.spring_tgt = Some((end.x, end.y, end_heading));+ self.pos = (end.x, end.y);+ self.heading = end_heading;+ self.path = None;+ self.dist = 0.0;+ } else {+ let s: PathState = p.sample(self.dist);+ self.pos = (s.x, s.y);+ let desired = s.heading + std::f64::consts::PI;+ let max_step = 14.0 * dt;+ self.heading = rotate_toward(self.heading, desired, max_step);+ }+ } else if let Some(mut s) = self.spring {+ if let Some((tx, ty, th)) = self.spring_tgt {+ let substeps = 4;+ let sdt = dt / substeps as f64;+ for _ in 0..substeps {+ s.vx += (-spring_k * s.ox - spring_c * s.vx) * sdt;+ s.vy += (-spring_k * s.oy - spring_c * s.vy) * sdt;+ s.ox += s.vx * sdt;+ s.oy += s.vy * sdt;+ }+ self.pos = (tx + s.ox, ty + s.oy);+ self.heading = th;+ if s.ox.hypot(s.oy) < 0.3 && s.vx.hypot(s.vy) < 2.0 {+ self.pos = (tx, ty);+ self.spring = None;+ } else {+ self.spring = Some(s);+ }+ }+ }++ if let Some(t) = self.click_t {+ let next = t + dt * 4.0;+ self.click_t = if next >= 1.0 { None } else { Some(next) };+ }++ // Idle-hide with 180ms fade matching Windows reference.+ let idle_hide_ms = self.motion.idle_hide_ms;+ if idle_hide_ms > 0.0 {+ let moving = self.path.is_some() || self.spring.is_some() || self.click_t.is_some();+ if moving {+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ } else {+ self.idle_secs += dt;+ let fade_start = idle_hide_ms / 1000.0;+ let fade_end = fade_start + 0.18;+ if self.idle_secs > fade_end {+ self.idle_alpha = 0.0;+ } else if self.idle_secs > fade_start {+ let t = (self.idle_secs - fade_start) / 0.18;+ self.idle_alpha = 1.0 - t.clamp(0.0, 1.0);+ }+ }+ } else {+ self.idle_alpha = 1.0;+ }+ }++ fn apply_command(&mut self, cmd: OverlayCommand) {+ match cmd {+ OverlayCommand::MoveTo { x, y, end_heading_radians } => {+ let (x0, y0) = self.pos;+ let th0 = self.heading + std::f64::consts::PI;+ let th1 = end_heading_radians + std::f64::consts::PI;+ const CLICK_OFFSET: f64 = 16.0;+ const TURN_RADIUS: f64 = 80.0;+ let tx = x + end_heading_radians.cos() * CLICK_OFFSET;+ let ty = y + end_heading_radians.sin() * CLICK_OFFSET;+ let plan = PathPlanner::plan(+ x0, y0, th0,+ tx, ty, th1,+ end_heading_radians,+ TURN_RADIUS,+ );+ self.path = Some(plan);+ self.dist = 0.0;+ self.start_t = Instant::now();+ self.spring = None;+ self.spring_tgt = None;+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ }+ OverlayCommand::ClickPulse { x, y } => {+ self.pos = (x, y);+ self.click_t = Some(0.0);+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;+ }+ OverlayCommand::SetEnabled(v) => {+ self.visible = v;+ }+ OverlayCommand::SetMotion(m) => {+ self.motion = m;+ }+ OverlayCommand::SetPalette(p) => {+ self.palette = p;+ }+ OverlayCommand::PinAbove(wid) => {+ self.pinned_wid = Some(wid);+ }+ OverlayCommand::SetGradient { gradient_colors, bloom_color } => {+ self.gradient_colors = gradient_colors;+ self.bloom_override = bloom_color;+ }+ OverlayCommand::SetShape(shape) => {+ self.shape = shape;+ }+ OverlayCommand::ShowFocusRect(_) => {}+ }+ }+}++fn rotate_toward(current: f64, desired: f64, max_step: f64) -> f64 {+ let mut diff = desired - current;+ while diff > std::f64::consts::PI { diff -= 2.0 * std::f64::consts::PI; }+ while diff < -std::f64::consts::PI { diff += 2.0 * std::f64::consts::PI; }+ current + diff.clamp(-max_step, max_step)+}++// ── tiny-skia render (shared logic) ──────────────────────────────────────++fn render_frame(rs: &RenderState) -> tiny_skia::Pixmap {+ let w = rs.virt_w.max(1) as u32;+ let h = rs.virt_h.max(1) as u32;+ let mut pm = tiny_skia::Pixmap::new(w, h)+ .unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap());++ if !rs.visible || rs.pos.0 < -100.0 || rs.idle_alpha < 0.004 {+ return pm;+ }++ let (px, py) = (+ rs.pos.0 - rs.virt_x as f64,+ rs.pos.1 - rs.virt_y as f64,+ );+ let heading = rs.heading;+ let alpha_scale = rs.idle_alpha as f32;++ // Bloom — use runtime bloom_override if set.+ let bloom_r: f32 = 22.0;+ let (br, bg, bb) = if let Some([r, g, b, _]) = rs.bloom_override {+ (r, g, b)+ } else {+ let [r, g, b, _] = rs.palette.bloom_inner; (r, g, b)+ };+ let (or_, og, ob) = if let Some([r, g, b, _]) = rs.bloom_override {+ (r, g, b)+ } else {+ let [r, g, b, _] = rs.palette.bloom_outer; (r, g, b)+ };+ let bloom_inner = tiny_skia::Color::from_rgba8(br, bg, bb, (115.0 * alpha_scale) as u8);+ let bloom_outer = tiny_skia::Color::from_rgba8(or_, og, ob, (26.0 * alpha_scale) as u8);+ let bloom_zero = tiny_skia::Color::from_rgba8(or_, og, ob, 0);++ let bloom_paint = {+ let mut p = tiny_skia::Paint::default();+ p.shader = tiny_skia::RadialGradient::new(+ tiny_skia::Point::from_xy(px as f32, py as f32),+ tiny_skia::Point::from_xy(px as f32, py as f32),+ bloom_r,+ vec![+ tiny_skia::GradientStop::new(0.0, bloom_inner),+ tiny_skia::GradientStop::new(0.5, bloom_outer),+ tiny_skia::GradientStop::new(1.0, bloom_zero),+ ],+ tiny_skia::SpreadMode::Pad,+ tiny_skia::Transform::identity(),+ ).unwrap_or(tiny_skia::Shader::SolidColor(bloom_inner));+ p.anti_alias = true;+ p+ };++ if let Some(r) = tiny_skia::Rect::from_xywh(+ (px - bloom_r as f64) as f32, (py - bloom_r as f64) as f32,+ bloom_r * 2.0, bloom_r * 2.0,+ ) {+ pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None);+ }++ // Click pulse.+ if let Some(t) = rs.click_t {+ let ring_r = (bloom_r + 20.0 * t as f32) * (1.0 - t as f32 * 0.5);+ let alpha = ((1.0 - t) * 180.0 * alpha_scale as f64) as u8;+ let [cr, cg, cb, _] = rs.palette.cursor_mid;+ let ring_color = tiny_skia::Color::from_rgba8(cr, cg, cb, alpha);+ let mut ring_paint = tiny_skia::Paint::default();+ ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color);+ ring_paint.anti_alias = true;+ let stroke = tiny_skia::Stroke { width: 2.0, ..Default::default() };+ let mut pb = tiny_skia::PathBuilder::new();+ pb.push_circle(px as f32, py as f32, ring_r);+ if let Some(path) = pb.finish() {+ pm.stroke_path(&path, &ring_paint, &stroke, tiny_skia::Transform::identity(), None);+ }+ }++ // Arrow / custom shape.+ if let Some(ref shape) = rs.shape {+ let sz = 32.0_f32;+ if let Some(pix) = tiny_skia::PixmapRef::from_bytes(&shape.pixels, shape.width, shape.height) {+ let transform = tiny_skia::Transform::from_rotate_at(+ heading.to_degrees() as f32 + 180.0,+ px as f32, py as f32,+ ).pre_translate(px as f32 - sz / 2.0, py as f32 - sz / 2.0);+ let mut paint = tiny_skia::PixmapPaint::default();+ paint.opacity = alpha_scale;+ pm.draw_pixmap(0, 0, pix, &paint, transform, None);+ }+ } else {+ let grad_override = if rs.gradient_colors.is_empty() { None } else { Some(&rs.gradient_colors) };+ draw_default_arrow(&mut pm, &rs.palette, grad_override, px as f32, py as f32, heading as f32, alpha_scale);+ }++ pm+}++fn draw_default_arrow(+ pm: &mut tiny_skia::Pixmap,+ palette: &Palette,+ gradient_override: Option<&Vec<[u8; 4]>>,+ px: f32, py: f32,+ heading: f32,+ alpha_scale: f32,+) {+ let verts: [(f32, f32); 4] = [(14.0, 0.0), (-8.0, -9.0), (-3.0, 0.0), (-8.0, 9.0)];+ let angle = heading + std::f64::consts::PI as f32;+ let (sa, ca) = (angle.sin(), angle.cos());+ let transform_pt = |(vx, vy): (f32, f32)| -> (f32, f32) {+ (px + ca * vx - sa * vy, py + sa * vx + ca * vy)+ };+ let pts: Vec<(f32, f32)> = verts.iter().map(|&v| transform_pt(v)).collect();+ let mut pb = tiny_skia::PathBuilder::new();+ pb.move_to(pts[0].0, pts[0].1);+ for p in &pts[1..] { pb.line_to(p.0, p.1); }+ pb.close();+ let arrow_path = match pb.finish() { Some(p) => p, None => return };++ let tip = pts[0];+ let tail = ((pts[1].0 + pts[3].0) / 2.0, (pts[1].1 + pts[3].1) / 2.0);+ let (r0, g0, b0) = if let Some(g) = gradient_override.and_then(|g| g.first()) {+ (g[0], g[1], g[2])+ } else { let [r, g, b, _] = palette.cursor_start; (r, g, b) };+ let (r1, g1, b1) = if let Some(g) = gradient_override.and_then(|g| g.get(1).or_else(|| g.first())) {+ (g[0], g[1], g[2])+ } else { let [r, g, b, _] = palette.cursor_mid; (r, g, b) };+ let (r2, g2, b2) = if let Some(g) = gradient_override.and_then(|g| g.last()) {+ (g[0], g[1], g[2])+ } else { let [r, g, b, _] = palette.cursor_end; (r, g, b) };+ let a = (255.0 * alpha_scale) as u8;++ let fill_paint = {+ let mut p = tiny_skia::Paint::default();+ p.shader = tiny_skia::LinearGradient::new(+ tiny_skia::Point::from_xy(tip.0, tip.1),+ tiny_skia::Point::from_xy(tail.0, tail.1),+ vec![+ tiny_skia::GradientStop::new(0.00, tiny_skia::Color::from_rgba8(r0, g0, b0, a)),+ tiny_skia::GradientStop::new(0.53, tiny_skia::Color::from_rgba8(r1, g1, b1, a)),+ tiny_skia::GradientStop::new(1.00, tiny_skia::Color::from_rgba8(r2, g2, b2, a)),+ ],+ tiny_skia::SpreadMode::Pad,+ tiny_skia::Transform::identity(),+ ).unwrap_or(tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(r1, g1, b1, a)));+ p.anti_alias = true;+ p+ };+ pm.fill_path(&arrow_path, &fill_paint, tiny_skia::FillRule::Winding,+ tiny_skia::Transform::identity(), None);++ let mut stroke_paint = tiny_skia::Paint::default();+ stroke_paint.shader = tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(255, 255, 255, a));+ stroke_paint.anti_alias = true;+ let stroke = tiny_skia::Stroke { width: 1.5, ..Default::default() };+ pm.stroke_path(&arrow_path, &stroke_paint, &stroke, tiny_skia::Transform::identity(), None);+}++// ── Win32 message-loop thread ─────────────────────────────────────────────++#[cfg(target_os = "windows")]+fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayCommand>) {+ use windows::Win32::Foundation::*;+ use windows::Win32::UI::WindowsAndMessaging::*;+ use windows::Win32::Graphics::Gdi::*;+ use windows::Win32::Media::timeBeginPeriod;+ use windows::Win32::System::LibraryLoader::GetModuleHandleW;+ use windows::core::PCWSTR;++ // Raise multimedia timer resolution to 1ms so SetTimer can deliver+ // WM_TIMER messages at ~8ms intervals (default is ~15ms).+ // Mirrors `_timerResolutionRaised = timeBeginPeriod(1) == 0` in the+ // .NET reference (AgentCursorOverlay.cs).+ unsafe { let _ = timeBeginPeriod(1); }++ // Collect virtual screen bounds (all monitors).+ let virt_x = unsafe { GetSystemMetrics(SM_XVIRTUALSCREEN) };+ let virt_y = unsafe { GetSystemMetrics(SM_YVIRTUALSCREEN) };+ let virt_w = unsafe { GetSystemMetrics(SM_CXVIRTUALSCREEN) };+ let virt_h = unsafe { GetSystemMetrics(SM_CYVIRTUALSCREEN) };++ // Update render state with virtual screen bounds.+ {+ let mut guard = RENDER.lock().unwrap();+ if let Some(rs) = guard.as_mut() {+ rs.virt_x = virt_x;+ rs.virt_y = virt_y;+ rs.virt_w = virt_w;+ rs.virt_h = virt_h;+ }+ }++ // Register window class.+ let class_name_w: Vec<u16> = "TropeCUA.AgentCursorOverlay\0".encode_utf16().collect();+ let title_w: Vec<u16> = format!("TropeCUA.AgentCursorOverlay.{}\0", cfg.cursor_id)+ .encode_utf16().collect();++ let hinstance = unsafe { GetModuleHandleW(PCWSTR::null()).unwrap_or_default() };++ let wc = WNDCLASSEXW {+ cbSize: std::mem::size_of::<WNDCLASSEXW>() as u32,+ style: CS_HREDRAW | CS_VREDRAW,+ lpfnWndProc: Some(wnd_proc),+ hInstance: hinstance.into(),+ lpszClassName: PCWSTR(class_name_w.as_ptr()),+ ..Default::default()+ };+ unsafe { RegisterClassExW(&wc); } // ignore error if already registered++ // WS_EX_TRANSPARENT | WS_EX_LAYERED | WS_EX_NOACTIVATE | WS_EX_TOOLWINDOW+ let ex_style = WS_EX_TRANSPARENT | WS_EX_LAYERED | WS_EX_NOACTIVATE | WS_EX_TOOLWINDOW;+ let style = WS_POPUP;++ let hwnd = unsafe {+ CreateWindowExW(+ ex_style,+ PCWSTR(class_name_w.as_ptr()),+ PCWSTR(title_w.as_ptr()),+ style,+ virt_x, virt_y, virt_w, virt_h,+ None, None,+ hinstance,+ None,+ )+ };++ if hwnd.is_err() {+ tracing::error!("Win32 overlay: CreateWindowExW failed");+ return;+ }+ let hwnd = hwnd.unwrap();++ // Show without activation (mirrors ShowWithoutActivation in C# ref).+ unsafe { ShowWindow(hwnd, SW_SHOWNOACTIVATE); }++ // Set up timer at 8ms (~125 Hz) matching the C# reference.+ unsafe { SetTimer(hwnd, 1, 8, None); }++ // Store hwnd and rx globally for the wnd_proc callback.+ OVERLAY_HWND.store(hwnd.0 as isize, std::sync::atomic::Ordering::Relaxed);+ *CMD_RX_WIN.lock().unwrap() = Some(rx);+ LAST_ZTICK.store(0, std::sync::atomic::Ordering::Relaxed);++ // Standard Win32 message loop.+ let mut msg = MSG::default();+ unsafe {+ while GetMessageW(&mut msg, None, 0, 0).as_bool() {+ TranslateMessage(&msg);+ DispatchMessageW(&msg);+ }+ }+}++#[cfg(not(target_os = "windows"))]+fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver<OverlayCommand>) {+ // No-op on non-Windows targets (cross-compile guard).+}++// ── Win32 globals (only used on Windows) ─────────────────────────────────++static OVERLAY_HWND: std::sync::atomic::AtomicIsize =+ std::sync::atomic::AtomicIsize::new(0);+static CMD_RX_WIN: Mutex<Option<std::sync::mpsc::Receiver<OverlayCommand>>> = Mutex::new(None);+static LAST_ZTICK: std::sync::atomic::AtomicU64 =+ std::sync::atomic::AtomicU64::new(0);++// ── Window procedure ──────────────────────────────────────────────────────++#[cfg(target_os = "windows")]+unsafe extern "system" fn wnd_proc(+ hwnd: windows::Win32::Foundation::HWND,+ msg: u32,+ wparam: windows::Win32::Foundation::WPARAM,+ lparam: windows::Win32::Foundation::LPARAM,+) -> windows::Win32::Foundation::LRESULT {+ use windows::Win32::Foundation::*;+ use windows::Win32::UI::WindowsAndMessaging::*;+ use windows::Win32::Graphics::Gdi::*;++ match msg {+ WM_TIMER => {+ let now_ms = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default()+ .as_millis() as u64;++ // Drain commands and tick animation. Measure real dt from last+ // tick — Windows timer resolution defaults to 15ms so the+ // hardcoded 8ms was running the animation at half speed.+ let pixmap = {+ let mut guard = RENDER.lock().unwrap();+ if let Some(rs) = guard.as_mut() {+ // Drain the channel.+ if let Ok(mut rx_guard) = CMD_RX_WIN.try_lock() {+ if let Some(ref rx) = *rx_guard {+ while let Ok(cmd) = rx.try_recv() {+ rs.apply_command(cmd);+ }+ }+ }+ let now = std::time::Instant::now();+ let dt = now.duration_since(rs.last_tick).as_secs_f64().clamp(0.0, 0.05);+ rs.last_tick = now;+ rs.tick(dt);+ Some(render_frame(rs))+ } else {+ None+ }+ };++ if let Some(pm) = pixmap {+ update_layered_window(hwnd, &pm);+ }++ // Z-order maintenance every 80ms.+ let last = LAST_ZTICK.load(std::sync::atomic::Ordering::Relaxed);+ if now_ms.wrapping_sub(last) >= 80 {+ LAST_ZTICK.store(now_ms, std::sync::atomic::Ordering::Relaxed);+ reapply_z_order(hwnd);+ }++ LRESULT(0)+ }+ WM_DESTROY => {+ PostQuitMessage(0);+ LRESULT(0)+ }+ _ => DefWindowProcW(hwnd, msg, wparam, lparam),+ }+}++// ── UpdateLayeredWindow helper ────────────────────────────────────────────++#[cfg(target_os = "windows")]+unsafe fn update_layered_window(+ hwnd: windows::Win32::Foundation::HWND,+ pixmap: &tiny_skia::Pixmap,+) {+ use windows::Win32::Foundation::*;+ use windows::Win32::Graphics::Gdi::*;+ use windows::Win32::UI::WindowsAndMessaging::{UpdateLayeredWindow, ULW_ALPHA};++ let w = pixmap.width() as i32;+ let h = pixmap.height() as i32;+ if w <= 0 || h <= 0 { return; }++ let hdc_screen = GetDC(None);+ let hdc_mem = CreateCompatibleDC(hdc_screen);++ // Create a 32-bit top-down DIB section (BGRA).+ let mut bmi = BITMAPINFO {+ bmiHeader: BITMAPINFOHEADER {+ biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,+ biWidth: w,+ biHeight: -h, // negative = top-down+ biPlanes: 1,+ biBitCount: 32,+ biCompression: BI_RGB.0,+ ..Default::default()+ },+ ..Default::default()+ };++ let mut bits_ptr = std::ptr::null_mut::<std::ffi::c_void>();+ let hbmp = CreateDIBSection(+ hdc_mem,+ &bmi,+ DIB_RGB_COLORS,+ &mut bits_ptr,+ None,+ 0,+ );+ if hbmp.is_err() || bits_ptr.is_null() {+ DeleteDC(hdc_mem);+ ReleaseDC(None, hdc_screen);+ return;+ }+ let hbmp = hbmp.unwrap();+ SelectObject(hdc_mem, hbmp);++ // Copy pixels: tiny-skia produces premultiplied RGBA; Win32 expects premultiplied BGRA.+ let src = pixmap.data();+ let dst = std::slice::from_raw_parts_mut(bits_ptr as *mut u8, (w * h * 4) as usize);+ for i in 0..(w * h) as usize {+ let r = src[i * 4];+ let g = src[i * 4 + 1];+ let b = src[i * 4 + 2];+ let a = src[i * 4 + 3];+ // Swap R <-> B for BGRA.+ dst[i * 4] = b;+ dst[i * 4 + 1] = g;+ dst[i * 4 + 2] = r;+ dst[i * 4 + 3] = a;+ }++ // UpdateLayeredWindow.+ let virt_x;+ let virt_y;+ {+ let guard = RENDER.lock().unwrap();+ if let Some(rs) = &*guard {+ virt_x = rs.virt_x;+ virt_y = rs.virt_y;+ } else {+ virt_x = 0;+ virt_y = 0;+ }+ }++ let pt_src = POINT { x: 0, y: 0 };+ let pt_dst = POINT { x: virt_x, y: virt_y };+ let sz = SIZE { cx: w, cy: h };+ let blend = BLENDFUNCTION {+ BlendOp: 0, // AC_SRC_OVER+ BlendFlags: 0,+ SourceConstantAlpha: 255,+ AlphaFormat: 1, // AC_SRC_ALPHA+ };+ UpdateLayeredWindow(hwnd, hdc_screen, Some(&pt_dst), Some(&sz),+ hdc_mem, Some(&pt_src), COLORREF(0), Some(&blend), ULW_ALPHA);++ DeleteObject(hbmp);+ DeleteDC(hdc_mem);+ ReleaseDC(None, hdc_screen);+}++#[cfg(target_os = "windows")]+unsafe fn reapply_z_order(hwnd: windows::Win32::Foundation::HWND) {+ use windows::Win32::Foundation::HWND;+ use windows::Win32::UI::WindowsAndMessaging::*;++ // Read pinned_wid from render state (RENDER lock released before this is called).+ let pinned_wid = {+ let guard = RENDER.lock().unwrap();+ guard.as_ref().and_then(|rs| rs.pinned_wid)+ };++ // If a target window is pinned, place the overlay just above it.+ // Falls back to HWND_TOP when not set or when the target is gone.+ let insert_after = if let Some(wid) = pinned_wid {+ let target = HWND(wid as *mut _);+ if IsWindow(target).as_bool() {+ target+ } else {+ HWND_TOPMOST+ }+ } else {+ HWND_TOPMOST+ };++ let _ = SetWindowPos(+ hwnd,+ insert_after,+ 0, 0, 0, 0,+ SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW | SWP_NOOWNERZORDER,+ );+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rsnew file mode 100644index 0000000000..c1605e19de--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs@@ -0,0 +1,2708 @@+//! Real Windows tool implementations (compiled only on Windows).++use async_trait::async_trait;++/// Animate the agent cursor to (sx, sy) in screen coordinates and wait for the+/// glide to finish before returning. No-op when the overlay is not enabled.+///+/// On the very first call the cursor is at the off-screen initial position+/// (-200, -200). Animating from there would cause a jarring off-screen fly-in,+/// so we snap to the target with a ClickPulse first and skip the glide wait.+async fn overlay_glide_to(sx: f64, sy: f64) {+ if !crate::overlay::is_enabled() { return; }+ let pos = crate::overlay::current_position();+ if pos.0 < 0.0 && pos.1 < 0.0 {+ // Snap to target on first use; no animation to wait for.+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ return;+ }+ // Compute travel time from distance and speed (matches Swift: peak=900, avg~467 pts/sec).+ let dist = ((sx - pos.0).powi(2) + (sy - pos.1).powi(2)).sqrt();+ let avg_speed = 467.0_f64; // (300 + 900 + 200) / 3 ≈ Swift avg+ let ms = (dist / avg_speed * 1000.0).clamp(80.0, 600.0) as u64;+ // Always arrive at 45° (upper-left) — matches Swift's endAngleDegrees: 45.0.+ crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo {+ x: sx, y: sy, end_heading_radians: std::f64::consts::FRAC_PI_4,+ });+ tokio::time::sleep(std::time::Duration::from_millis(ms)).await;+}+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};+use serde_json::{json, Value};+use std::sync::{Arc, RwLock};++use crate::uia::ElementCache;+use cursor_overlay::CursorRegistry;+use windows::core::Interface as _;++// ── DriverConfig + ResizeRegistry + ZoomRegistry ─────────────────────────────++#[derive(Clone)]+pub struct DriverConfig {+ pub capture_mode: String,+ pub max_image_dimension: u32,+}++impl Default for DriverConfig {+ fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } }+}++pub struct ResizeRegistry {+ ratios: std::sync::Mutex<std::collections::HashMap<u32, f64>>,+}++impl ResizeRegistry {+ pub fn new() -> Self { Self { ratios: std::sync::Mutex::new(Default::default()) } }+ pub fn set_ratio(&self, pid: u32, ratio: f64) { self.ratios.lock().unwrap().insert(pid, ratio); }+ pub fn clear_ratio(&self, pid: u32) { self.ratios.lock().unwrap().remove(&pid); }+ pub fn ratio(&self, pid: u32) -> Option<f64> { self.ratios.lock().unwrap().get(&pid).copied() }+}++/// Per-process zoom context — stores padded crop origin and resize scale from+/// the most recent `zoom` call so `click(from_zoom=true)` can translate+/// zoom-image pixel coordinates back to full-window coordinates.+#[derive(Clone, Copy, Debug)]+pub struct ZoomContext {+ pub origin_x: f64,+ pub origin_y: f64,+ /// Inverse resize scale: `cw / out_w` (1.0 = no downscale).+ pub scale_inv: f64,+}++impl ZoomContext {+ pub fn zoom_to_window(&self, px: f64, py: f64) -> (f64, f64) {+ (self.origin_x + px * self.scale_inv, self.origin_y + py * self.scale_inv)+ }+}++pub struct ZoomRegistry {+ inner: std::sync::Mutex<std::collections::HashMap<u32, ZoomContext>>,+}++impl ZoomRegistry {+ pub fn new() -> Self { Self { inner: std::sync::Mutex::new(Default::default()) } }+ pub fn set(&self, pid: u32, ctx: ZoomContext) { self.inner.lock().unwrap().insert(pid, ctx); }+ pub fn get(&self, pid: u32) -> Option<ZoomContext> { self.inner.lock().unwrap().get(&pid).copied() }+}++pub struct ToolState {+ pub element_cache: Arc<ElementCache>,+ pub cursor_registry: Arc<CursorRegistry>,+ pub resize_registry: Arc<ResizeRegistry>,+ pub zoom_registry: Arc<ZoomRegistry>,+ pub config: Arc<RwLock<DriverConfig>>,+}++impl ToolState {+ pub fn new() -> Arc<Self> {+ Arc::new(Self {+ element_cache: Arc::new(ElementCache::new()),+ cursor_registry: Arc::new(CursorRegistry::new()),+ resize_registry: Arc::new(ResizeRegistry::new()),+ zoom_registry: Arc::new(ZoomRegistry::new()),+ config: Arc::new(RwLock::new(DriverConfig::default())),+ })+ }+}++// ── list_apps ────────────────────────────────────────────────────────────────++pub struct ListAppsTool;++static LIST_APPS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ListAppsTool {+ fn def(&self) -> &ToolDef {+ LIST_APPS_DEF.get_or_init(|| ToolDef {+ name: "list_apps".into(),+ // Description copied from Swift `ListAppsTool.swift` and adapted —+ // notes Windows-specific limitations (no bundle_id, no installed+ // enumeration yet). Same semantics for running/active state flags.+ description: "List Windows apps that are currently running, with per-app state flags:\n\n\+ - running: is a process for this app live? (pid is 0 when false)\n\+ - active: is it the system-frontmost app? (implies running)\n\n\+ Only processes that own at least one visible top-level window are included — \+ background services and console processes are filtered out (mirrors Swift's \+ NSApplicationActivationPolicyRegular filter).\n\n\+ Use this for \"is X running?\". For per-window state — on-screen, minimized, \+ window titles — call list_windows instead. For just opening an app — running \+ or not — call launch_app({path: ...}) directly; list_apps is not a prerequisite.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, _args: Value) -> ToolResult {+ // Strategy (port of `AppEnumerator.apps()` semantics):+ // 1. Enumerate top-level visible windows; their owner pids are the+ // "running apps with a user-facing surface".+ // 2. Look up each pid's executable name via the process table.+ // 3. Mark `active` for the pid that owns GetForegroundWindow.+ let apps = tokio::task::spawn_blocking(|| -> Vec<serde_json::Value> {+ use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow;+ use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId;++ let wins = crate::win32::list_windows(None);+ // Deduplicate by pid — one app may own several top-level windows.+ let mut seen_pids = std::collections::HashSet::new();+ let mut running_pids: Vec<u32> = Vec::new();+ for w in &wins {+ if seen_pids.insert(w.pid) { running_pids.push(w.pid); }+ }++ let foreground_pid = unsafe {+ let fg = GetForegroundWindow();+ let mut pid = 0u32;+ let _ = GetWindowThreadProcessId(fg, Some(&mut pid));+ pid+ };++ let procs = crate::win32::list_processes();+ let pid_to_name: std::collections::HashMap<u32, &str> =+ procs.iter().map(|p| (p.pid, p.name.as_str())).collect();++ running_pids.iter().map(|&pid| {+ let name = pid_to_name.get(&pid).copied().unwrap_or("<unknown>");+ json!({+ "pid": pid,+ "bundle_id": serde_json::Value::Null, // Windows has no bundle ID concept+ "name": name,+ "running": true,+ "active": pid == foreground_pid,+ })+ }).collect()+ }).await.unwrap_or_default();++ let running_count = apps.iter().filter(|a| a["running"].as_bool().unwrap_or(false)).count();+ let total = apps.len();+ let installed_only = total - running_count;+ // Match Swift's text format: header line + bullet per running app.+ let mut lines = vec![format!(+ "✅ Found {total} app(s): {running_count} running, {installed_only} installed-not-running."+ )];+ for app in apps.iter().filter(|a| a["running"].as_bool().unwrap_or(false)) {+ let name = app["name"].as_str().unwrap_or("?");+ let pid = app["pid"].as_u64().unwrap_or(0);+ // Windows lacks bundle_id; omit the trailing [bundle] segment.+ lines.push(format!("- {name} (pid {pid})"));+ }+ // `apps` key (matches Swift `Output.apps`); `processes` key kept as+ // an alias for any pre-existing Rust callers that read the old shape.+ let structured = json!({+ "apps": apps,+ "processes": apps.iter().map(|a| json!({+ "pid": a["pid"], "name": a["name"]+ })).collect::<Vec<_>>(),+ });+ ToolResult::text(lines.join("\n")).with_structured(structured)+ }+}++// ── list_windows ─────────────────────────────────────────────────────────────++pub struct ListWindowsTool;++static LIST_WINDOWS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ListWindowsTool {+ fn def(&self) -> &ToolDef {+ LIST_WINDOWS_DEF.get_or_init(|| ToolDef {+ name: "list_windows".into(),+ // Description matches Swift `ListWindowsTool.swift` semantics —+ // Windows-specific caveat: no Spaces concept, so on_current_space+ // / space_ids are always omitted and current_space_id is null.+ description: "List every top-level window currently known to the window manager. \+ Each record self-contains its owning app identity so the caller never has to \+ join back against list_apps.\n\n\+ Use this — not list_apps — for any window-level reasoning: \"does this app \+ have a visible window right now?\", \"which of this pid's windows is the main \+ one?\".\n\n\+ Per-record fields: window_id (HWND), pid + app_name, title, \+ bounds {x, y, width, height}, layer (always 0), z_index (stacking order), \+ is_on_screen. The macOS-specific on_current_space / space_ids fields are \+ omitted on Windows; current_space_id is null.\n\n\+ Inputs: pid (optional pid filter), on_screen_only (bool, default false — \+ Windows currently only enumerates visible non-minimized windows; this flag \+ is accepted but has no effect on the result set yet).".into(),+ input_schema: json!({"type":"object","properties":{+ "pid":{"type":"integer","description":"Optional pid filter. When set, only this pid's windows are returned."},+ "on_screen_only":{"type":"boolean","description":"When true, drop windows that aren't currently on-screen. Default false."}+ },"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let filter_pid = args.get("pid").and_then(|v| v.as_u64()).map(|v| v as u32);+ let _on_screen_only = args.get("on_screen_only").and_then(|v| v.as_bool()).unwrap_or(false);+ let (windows, pid_to_name) = tokio::task::spawn_blocking(move || {+ let wins = crate::win32::list_windows(filter_pid);+ let procs = crate::win32::list_processes();+ let map: std::collections::HashMap<u32, String> =+ procs.into_iter().map(|p| (p.pid, p.name)).collect();+ (wins, map)+ }).await.unwrap_or_default();++ // Swift surfaces a warning when a pid filter matches nothing.+ if let Some(fp) = filter_pid {+ if windows.is_empty() {+ use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow;+ use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId;+ let fg_pid = unsafe {+ let fg = GetForegroundWindow();+ let mut p = 0u32;+ let _ = GetWindowThreadProcessId(fg, Some(&mut p));+ p+ };+ let fg_name = pid_to_name.get(&fg_pid).map(|s| s.as_str()).unwrap_or("?");+ let msg = format!(+ "⚠️ No windows found for pid {fp}. The pid may be wrong or the app may not \+ have created a window yet. The current frontmost app appears to be \+ \"{fg_name}\" (pid {fg_pid})."+ );+ return ToolResult::text(msg);+ }+ }++ // z_index: EnumWindows on Win32 returns top-to-bottom; higher index =+ // farther from front. Swift convention: higher z_index = closer to+ // front. Invert via `(len - 1 - i)`.+ let n = windows.len();+ let records: Vec<serde_json::Value> = windows.iter().enumerate().map(|(i, w)| {+ let app_name = pid_to_name.get(&w.pid).cloned().unwrap_or_default();+ let z_index = (n.saturating_sub(1).saturating_sub(i)) as i64;+ json!({+ "window_id": w.hwnd,+ "pid": w.pid,+ "app_name": app_name,+ "title": w.title,+ "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height },+ "layer": 0,+ "z_index": z_index,+ "is_on_screen": true,+ })+ }).collect();++ let total = records.len();+ let pid_count = windows.iter().map(|w| w.pid).collect::<std::collections::HashSet<_>>().len();+ let on_screen = records.iter().filter(|r| r["is_on_screen"].as_bool().unwrap_or(false)).count();+ // Swift text format 1:1 — append the SPI-unavailable note since+ // Windows has no equivalent of macOS Spaces.+ let header = format!(+ "✅ Found {total} window(s) across {pid_count} app(s); {on_screen} on-screen. \+ (SkyLight Space SPIs unavailable — on_current_space / space_ids omitted.)"+ );+ let mut lines = vec![header];+ for r in &records {+ let app = r["app_name"].as_str().unwrap_or("?");+ let pid = r["pid"].as_u64().unwrap_or(0);+ let tt = r["title"].as_str().unwrap_or("");+ let wid = r["window_id"].as_u64().unwrap_or(0);+ let title_disp = if tt.is_empty() { "(no title)".to_owned() } else { format!("\"{tt}\"") };+ let tag = if r["is_on_screen"].as_bool().unwrap_or(false) { "" } else { " [off-screen]" };+ lines.push(format!("- {app} (pid {pid}) {title_disp} [window_id: {wid}]{tag}"));+ }++ // Legacy alias: keep the old flat fields under each record for any+ // pre-existing Rust callers, but they read from the same source.+ let legacy_windows: Vec<serde_json::Value> = windows.iter().map(|w| json!({+ "window_id": w.hwnd, "pid": w.pid, "title": w.title,+ "x": w.x, "y": w.y, "width": w.width, "height": w.height,+ })).collect();+ let structured = json!({+ "windows": records,+ "current_space_id": serde_json::Value::Null, // Windows has no Spaces+ "_legacy_windows": legacy_windows,+ });+ ToolResult::text(lines.join("\n")).with_structured(structured)+ }+}++// ── get_window_state ─────────────────────────────────────────────────────────++pub struct GetWindowStateTool {+ state: Arc<ToolState>,+}++static GWS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetWindowStateTool {+ fn def(&self) -> &ToolDef {+ GWS_DEF.get_or_init(|| ToolDef {+ name: "get_window_state".into(),+ // Description ported from Swift `GetWindowStateTool.swift` with+ // Windows-specific adaptations (UIA instead of AX; no SkyLight+ // Space validation; no per-call `javascript` field — that's a+ // macOS AppleScript hook).+ description: "Walk a running app's UIA tree and return a Markdown rendering of its \+ UI, tagging every actionable element with [element_index N]. Pass those \+ indices to `click`, `type_text`, `scroll`, etc. — those tools resolve the \+ index to the cached element on the server.\n\n\+ INVARIANT: call `get_window_state` once per turn per (pid, window_id) before \+ any element-indexed action against that window. The index map is replaced by \+ the next snapshot of the same (pid, window_id).\n\n\+ The UIA tree walked is the window's tree (HWND-scoped); the screenshot and \+ window bounds reported come from the same `window_id`. This is the source of \+ truth for which window the caller intends to reason about — the driver never \+ picks a window implicitly.\n\n\+ `window_id` MUST belong to `pid`; the call returns `isError: true` otherwise. \+ The driver does not auto-fall-back to a different window.\n\n\+ Set `query` to a case-insensitive substring to filter `tree_markdown` down to \+ matching lines plus their ancestor chain. element_index values and \+ `element_count` are unchanged — the filter only trims the rendered Markdown.\n\n\+ Response shape is controlled by `capture_mode`:\n\+ - `som` (default) — walks UIA AND captures screenshot.\n\+ - `vision` — captures only; UIA walk skipped; element_index cache not updated.\n\+ - `ax` — UIA only; no screenshot.\n\n\+ Uses `IUIAutomationCacheRequest` to batch-fetch all element properties in a \+ single COM call (Chrome's ~5000-element tree returns in ~2-3s instead of \+ timing out at 4s with per-property RPCs).\n\n\+ Windows requires no special permissions.".into(),+ input_schema: json!({"type":"object","required":["pid","window_id"],"properties":{+ "pid":{"type":"integer","description":"Process ID from `list_apps`."},+ "window_id":{"type":"integer","description":"HWND of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array."},+ "capture_mode":{"type":"string","enum":["som","vision","ax"],+ "description":"som=tree+screenshot (default), vision=screenshot only, ax=tree only."},+ "query":{"type":"string","description":"Optional case-insensitive substring. When set, `tree_markdown` only contains lines that match plus their ancestor chain; element indices and `element_count` are unchanged."}+ },"additionalProperties":false}),+ // Swift annotation: idempotent: false (each call is a fresh snapshot).+ read_only: true, destructive: false, idempotent: false, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let hwnd = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v,+ None => return ToolResult::error(+ "Missing required integer field window_id. Use `list_windows` to enumerate \+ the target app's windows, or read `launch_app`'s `windows` array."),+ };+ // Validate window belongs to pid — Swift's hard error.+ let windows_for_pid = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))+ .await.unwrap_or_default();+ if !windows_for_pid.iter().any(|w| w.hwnd == hwnd) {+ // Check if the window exists under a different pid.+ let all = tokio::task::spawn_blocking(|| crate::win32::list_windows(None))+ .await.unwrap_or_default();+ if let Some(w) = all.iter().find(|w| w.hwnd == hwnd) {+ return ToolResult::error(format!(+ "window_id {hwnd} belongs to pid {}, not pid {pid}. Call \+ `list_windows({{\"pid\": {pid}}})` to get this pid's own windows.",+ w.pid));+ }+ return ToolResult::error(format!(+ "No window with window_id {hwnd} exists. Call `list_windows({{\"pid\": \+ {pid}}})` for candidates."));+ }+ let (default_mode, max_dim) = {+ let cfg = self.state.config.read().unwrap();+ (cfg.capture_mode.clone(), cfg.max_image_dimension)+ };+ let capture_mode = args.get("capture_mode").and_then(|v| v.as_str())+ .unwrap_or(&default_mode).to_owned();+ let query = args.get("query").and_then(|v| v.as_str()).map(str::to_owned);++ // "ax" = tree only; "vision" = screenshot only; "som" (default) = both.+ let do_tree = capture_mode != "vision";+ let do_shot = capture_mode != "ax";++ let state = self.state.clone();+ let q = query.clone();+ let blocking = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {+ let tree_result = if do_tree {+ Some(crate::uia::walk_tree(hwnd, q.as_deref()))+ } else {+ None+ };+ let screenshot = if do_shot {+ match crate::capture::screenshot_window_bytes(hwnd) {+ Ok(raw) => {+ let orig_w = crate::capture::png_dimensions_pub(&raw).map(|(w, _)| w).unwrap_or(0);+ let png = crate::capture::resize_png_if_needed(&raw, max_dim)?;+ let (w, h) = crate::capture::png_dimensions_pub(&png)?;+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};+ let original_w = if w < orig_w { Some(orig_w) } else { None };+ Some((B64.encode(&png), w, h, original_w))+ }+ Err(_) => None,+ }+ } else {+ None+ };+ Ok((tree_result, screenshot))+ });+ // Timeout: Chrome's UIA provider can block indefinitely on property reads.+ let result: Result<anyhow::Result<_>, _> = match tokio::time::timeout(+ std::time::Duration::from_secs(4), blocking).await+ {+ Ok(join_result) => join_result.map_err(|e| anyhow::anyhow!("task panic: {e}")),+ Err(_elapsed) => Err(anyhow::anyhow!("get_window_state timed out after 4s (UIA provider unresponsive)")),+ };+ let result = result.and_then(|r| r);++ match result {+ Ok((tree_opt, screenshot_opt)) => {+ let mut content = Vec::new();+ let mut structured = json!({ "window_id": hwnd, "pid": pid });++ if let Some(tr) = tree_opt {+ let count = tr.nodes.iter().filter(|n| n.element_index.is_some()).count();+ let header = format!("window_id={hwnd} pid={pid} elements={count}\n\n");+ content.push(mcp_server::protocol::Content::text(header + &tr.tree_markdown));+ state.element_cache.update(pid, hwnd, &tr.nodes);+ structured["element_count"] = json!(count);+ structured["tree_markdown"] = json!(tr.tree_markdown);+ }++ if let Some((b64, w, h, orig_w)) = screenshot_opt {+ if let Some(ow) = orig_w {+ if w > 0 { state.resize_registry.set_ratio(pid, ow as f64 / w as f64); }+ } else {+ state.resize_registry.clear_ratio(pid);+ }+ content.push(mcp_server::protocol::Content::image_png(b64));+ structured["screenshot_width"] = json!(w);+ structured["screenshot_height"] = json!(h);+ }++ ToolResult { content, is_error: None, structured_content: Some(structured) }+ }+ Err(e) => ToolResult::error(format!("Error: {e}")),+ }+ }+}++// ── launch_app ───────────────────────────────────────────────────────────────++pub struct LaunchAppTool;+static LAUNCH_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for LaunchAppTool {+ fn def(&self) -> &ToolDef {+ LAUNCH_DEF.get_or_init(|| ToolDef {+ name: "launch_app".into(),+ // Description ported from Swift `LaunchAppTool.swift` with+ // Windows-specific notes. `bundle_id` is accepted as an alias+ // for `name` (Windows has no bundle-identifier concept).+ description: "Launch a Windows app hidden — the driver never brings the target to \+ the foreground, the target's window is launched with SW_SHOWNOACTIVATE so it \+ does not steal focus from whatever is currently frontmost.\n\n\+ Provide either `bundle_id` / `name` (resolved via ShellExecuteEx's PATH search) \+ or `path` (full path to an executable). If both `name` and `path` are given, \+ `path` wins. `urls` opens each URL in the default browser without activating it.\n\n\+ Returns the launched app's pid, name, active flag, AND a `windows` array — \+ same per-window shape `list_windows` returns. When the launch settles but no \+ window has materialized yet (transient; rare), `windows` comes back empty — \+ call `list_windows(pid)` explicitly a moment later.\n\n\+ Windows-only field: `path` (Swift uses `bundle_id` since macOS apps resolve via \+ LaunchServices; Windows has no LaunchServices, so `path` is the canonical form). \+ The macOS-specific `electron_debugging_port`, `webkit_inspector_port`, \+ `creates_new_application_instance`, and `additional_arguments` fields are \+ accepted; `additional_arguments` is honored, the others currently no-op on \+ Windows.".into(),+ input_schema: json!({"type":"object","properties":{+ "bundle_id":{"type":"string","description":"Alias for `name` on Windows (no bundle-id concept). Either bundle_id or name must be provided when path is absent."},+ "name":{"type":"string","description":"App display name passed to ShellExecuteEx."},+ "path":{"type":"string","description":"Full path to executable. Windows-only; takes precedence over name/bundle_id."},+ "urls":{"type":"array","items":{"type":"string"},"description":"URLs to open in the default browser via ShellExecuteEx (no activation)."},+ "additional_arguments":{"type":"array","items":{"type":"string"},"description":"Extra command-line arguments passed to the launched process."},+ "electron_debugging_port":{"type":"integer","description":"Accepted for cross-platform parity; currently no-op on Windows."},+ "webkit_inspector_port":{"type":"integer","description":"Accepted for cross-platform parity; no-op on Windows."},+ "creates_new_application_instance":{"type":"boolean","description":"Accepted for parity; no-op on Windows (ShellExecuteEx always creates a new process)."}+ },"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let path_opt = args.get("path").and_then(|v| v.as_str()).map(str::to_owned);+ let name_opt = args.get("name").and_then(|v| v.as_str()).map(str::to_owned);+ let bundle_id_opt = args.get("bundle_id").and_then(|v| v.as_str()).map(str::to_owned);+ let urls: Vec<String> = args.get("urls").and_then(|v| v.as_array())+ .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();+ let extra_args: Vec<String> = args.get("additional_arguments").and_then(|v| v.as_array())+ .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();++ // Resolve target — path > name > bundle_id (alias of name on Windows).+ let target = path_opt.clone()+ .or(name_opt.clone())+ .or(bundle_id_opt.clone());++ if target.is_none() && urls.is_empty() {+ // Match Swift's wording verbatim.+ return ToolResult::error("Provide either bundle_id or name to identify the app to launch.");+ }++ // Single launch path: use ShellExecuteExW with SEE_MASK_NOCLOSEPROCESS+ // so we can read the spawned process's pid back.+ let display = target.clone().unwrap_or_else(|| urls.join(", "));+ let extra_joined = extra_args.join(" ");+ let urls_clone = urls.clone();+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<u32> {+ use windows::Win32::UI::Shell::{+ ShellExecuteExW, SHELLEXECUTEINFOW, SEE_MASK_NOCLOSEPROCESS,+ };+ use windows::Win32::UI::WindowsAndMessaging::SW_SHOWNOACTIVATE;+ use windows::Win32::System::Threading::GetProcessId;+ use windows::Win32::Foundation::CloseHandle;+ use windows::core::PCWSTR;++ fn to_wide(s: &str) -> Vec<u16> {+ s.encode_utf16().chain(std::iter::once(0)).collect()+ }+ let op_w = to_wide("open");+ let file_w = to_wide(if let Some(t) = target.as_deref() { t } else { urls_clone.first().map(|s| s.as_str()).unwrap_or("") });+ let args_w = to_wide(&extra_joined);++ let mut info = SHELLEXECUTEINFOW {+ cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32,+ fMask: SEE_MASK_NOCLOSEPROCESS,+ lpVerb: PCWSTR(op_w.as_ptr()),+ lpFile: PCWSTR(file_w.as_ptr()),+ lpParameters: if extra_joined.is_empty() { PCWSTR::null() } else { PCWSTR(args_w.as_ptr()) },+ nShow: SW_SHOWNOACTIVATE.0,+ ..Default::default()+ };+ unsafe { ShellExecuteExW(&mut info)?; }+ let pid = if !info.hProcess.is_invalid() {+ let p = unsafe { GetProcessId(info.hProcess) };+ unsafe { let _ = CloseHandle(info.hProcess); }+ p+ } else { 0 };++ // Open any additional URLs in the default browser (no focus+ // steal, no pid capture for these — Swift's NSWorkspace flow+ // behaves the same way for secondary URLs).+ for url in &urls_clone[urls_clone.len().min(1)..] {+ let file = to_wide(url);+ let mut url_info = SHELLEXECUTEINFOW {+ cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32,+ lpVerb: PCWSTR(op_w.as_ptr()),+ lpFile: PCWSTR(file.as_ptr()),+ nShow: SW_SHOWNOACTIVATE.0,+ ..Default::default()+ };+ unsafe { let _ = ShellExecuteExW(&mut url_info); }+ }++ Ok(pid)+ }).await;++ let pid = match result {+ Ok(Ok(p)) => p,+ Ok(Err(e)) => return ToolResult::error(format!("Failed to launch: {e}")),+ Err(e) => return ToolResult::error(format!("Task error: {e}")),+ };++ // Resolve the pid's windows so the caller can skip a list_windows+ // round-trip — same approach as Swift's `resolveWindows`. Retry+ // 5×200ms; Win32 window registration can lag the launch.+ let mut windows_json: Vec<serde_json::Value> = Vec::new();+ for _ in 0..5 {+ let wins = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))+ .await.unwrap_or_default();+ if !wins.is_empty() {+ windows_json = wins.iter().map(|w| json!({+ "window_id": w.hwnd, "title": w.title,+ "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height },+ "layer": 0,+ "z_index": 0, // single-window context; per-record z_index already in list_windows+ "is_on_screen": true,+ })).collect();+ break;+ }+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;+ }++ // Match Swift text format 1:1.+ let mut summary = format!("✅ Launched {display} (pid {pid}) in background.");+ if !windows_json.is_empty() {+ summary.push_str("\n\nWindows:");+ for w in &windows_json {+ let title = w["title"].as_str().unwrap_or("");+ let title_disp = if title.is_empty() { "(no title)".to_owned() } else { format!("\"{title}\"") };+ let wid = w["window_id"].as_u64().unwrap_or(0);+ summary.push_str(&format!("\n- {title_disp} [window_id: {wid}]"));+ }+ summary.push_str(&format!("\n→ Call get_window_state(pid: {pid}, window_id) to inspect."));+ }+ let structured = json!({+ "pid": pid,+ "bundle_id": serde_json::Value::Null,+ "name": display,+ "running": true,+ "active": false, // SW_SHOWNOACTIVATE — Swift's background-launch invariant+ "windows": windows_json,+ });+ ToolResult::text(summary).with_structured(structured)+ }+}++// ── click ─────────────────────────────────────────────────────────────────────++pub struct ClickTool {+ state: Arc<ToolState>,+}++static CLICK_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ClickTool {+ fn def(&self) -> &ToolDef {+ CLICK_DEF.get_or_init(|| ToolDef {+ name: "click".into(),+ // Description matches Swift `ClickTool.swift` semantics (left-click+ // primitive with two addressing modes — element_index via UIA, or+ // window-local pixel coords via PostMessage). Windows-only schema+ // extras (`button`, no `modifier`/`action`/`debug_image_out`) are+ // documented in PARITY.md.+ description: "Left-click against a target pid. Two addressing modes:\n\n\+ - `element_index` + `window_id` (from the last `get_window_state` snapshot \+ of that window) — performs the UIA Invoke pattern on the cached element via \+ PostMessage. No cursor move, no focus steal. Requires a prior \+ `get_window_state(pid, window_id)` in this turn; the element_index cache \+ is scoped per (pid, window_id).\n\n\+ - `x`, `y` (window-local screenshot pixels, top-left origin of the PNG \+ returned by `get_window_state`) — synthesizes mouse events via \+ PostMessage(WM_LBUTTONDOWN/UP) and delivers them to the deepest child \+ window under that point. `count: 2` posts two down/up pairs for a \+ double-click.\n\n\+ Exactly one of `element_index` or (`x` AND `y`) must be provided. \+ `pid` is required in both modes. `window_id` is required when \+ `element_index` is used (scopes the cache lookup). After a `zoom` call, \+ pass `from_zoom=true` to auto-translate zoom-image coords back to \+ full-window space.\n\n\+ Windows-only convenience: `button: \"left\"|\"right\"|\"middle\"` switches \+ the mouse button (Swift exposes right-click as a separate `right_click` \+ tool). The Swift-only `action` / `modifier` / `debug_image_out` schema \+ fields aren't supported yet.".into(),+ input_schema: json!({+ "type":"object","required":["pid"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "window_id":{"type":"integer","description":"HWND for the window whose get_window_state produced the element_index. Required when element_index is used."},+ "element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."},+ "x":{"type":"number","description":"X in window-local screenshot pixels — same space as the PNG get_window_state returns. Must be provided together with y."},+ "y":{"type":"number","description":"Y in window-local screenshot pixels. Must be provided together with x."},+ "button":{"type":"string","enum":["left","right","middle"],"description":"Mouse button. Windows-only convenience; Swift exposes right-click as a separate `right_click` tool."},+ "count":{"type":"integer","minimum":1,"maximum":3,"description":"Click count — 1 (single), 2 (double), 3 (triple). Default 1."},+ "from_zoom":{"type":"boolean","description":"When true, x and y are pixel coordinates in the last `zoom` image for this pid. The driver maps them back to window coords."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = match args.get("pid").and_then(|v| v.as_u64()) {+ Some(v) => v as u32,+ None => return ToolResult::error("Missing required parameter: pid"),+ };+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ let elem_idx = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let x = args.get("x").and_then(|v| v.as_f64());+ let y = args.get("y").and_then(|v| v.as_f64());+ let button = args.get("button").and_then(|v| v.as_str()).unwrap_or("left").to_owned();+ let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(1) as usize;++ // Resolve HWND: explicit, or auto from pid.+ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };++ if let Some(idx) = elem_idx {+ // UIA path: get cached center (no COM call needed — captured at walk time).+ let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {+ Some(v) => v,+ None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")),+ };+ // Step 2: animate cursor to screen coords (awaits glide_duration_ms).+ overlay_glide_to(cx as f64, cy as f64).await;+ // Step 3: click pulse + post click.+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {+ x: cx as f64, y: cy as f64,+ });+ let btn = button.clone();+ let action_name = match btn.as_str() {+ "right" => "ShowMenu", // closest UIA analogue of Swift "show_menu"+ "middle" => "Invoke",+ _ => "Invoke", // left-click = UIA Invoke pattern (matches Swift "AXPress")+ };+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {+ // cx/cy are screen coords from CurrentBoundingRectangle; use+ // post_click_screen so deepest-child resolution happens once.+ crate::input::post_click_screen(hwnd, cx, cy, count, &btn)?;+ // Match Swift text format: "✅ Performed AXPress on [N] role \"title\"."+ // (UIA has no readily-available role/name on the cached element here;+ // emit element_index + screen coords for traceability instead).+ Ok(format!("✅ Performed {action_name} on [{idx}] (screen ({cx},{cy}))."))+ }).await;+ match result {+ Ok(Ok(msg)) => ToolResult::text(msg),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ } else if let (Some(mut px), Some(mut py)) = (x, y) {+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);+ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(px, py); px = wx; py = wy; }+ None => return ToolResult::error(+ format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")+ ),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ px *= ratio;+ py *= ratio;+ }+ // px/py are window-client coords. Convert to screen for the overlay.+ let (sx, sy) = unsafe {+ use windows::Win32::Foundation::{HWND, POINT};+ use windows::Win32::Graphics::Gdi::ClientToScreen;+ let mut pt = POINT { x: px as i32, y: py as i32 };+ let _ = ClientToScreen(HWND(hwnd as *mut _), &mut pt);+ (pt.x as f64, pt.y as f64)+ };+ overlay_glide_to(sx, sy).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ let btn = button.clone();+ let result = tokio::task::spawn_blocking(move || {+ crate::input::post_click(hwnd, px as i32, py as i32, count, &btn)+ }).await;+ match result {+ Ok(Ok(())) => {+ // Match Swift text format: "✅ Posted click/double-click/triple-click to pid X."+ let click_word = match count {+ 2 => "double-click",+ 3 => "triple-click",+ _ => "click",+ };+ ToolResult::text(format!("✅ Posted {click_word} to pid {pid}."))+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ } else {+ // Match Swift's error wording for the missing-target case.+ ToolResult::error("Provide element_index or (x, y) to address the click target.")+ }+ }+}++// ── type_text ─────────────────────────────────────────────────────────────────++pub struct TypeTextTool {+ state: Arc<ToolState>,+}++static TYPE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for TypeTextTool {+ fn def(&self) -> &ToolDef {+ TYPE_DEF.get_or_init(|| ToolDef {+ name: "type_text".into(),+ // Description ported from Swift `TypeTextTool.swift` with the+ // Windows transport note. Swift tries AXSetAttribute(kAXSelectedText)+ // first, falls back to character-by-character CGEvent synthesis.+ // Windows uses character-by-character PostMessage(WM_CHAR) to the+ // window's focused child — there's no UIA equivalent of+ // AXSelectedText wired up yet, so it's always the synthesis path.+ description: "Insert text into the target pid via character-by-character \+ PostMessage(WM_CHAR) to the focused window. No focus steal.\n\n\+ Special keys (Return, Escape, arrows, Tab) go through `press_key` / \+ `hotkey` — they are not text.\n\n\+ Optional `element_index` + `window_id` (from the last `get_window_state` \+ snapshot of that window) is accepted for parity but currently no-op on \+ Windows (UIA SetFocus not wired up yet); without `element_index`, the \+ write targets the pid's focused window — same as Swift's no-element path.\n\n\+ `delay_ms` (0–200, default 30) spaces successive characters so \+ autocomplete and IME can keep up.".into(),+ input_schema: json!({+ "type":"object","required":["pid","text"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "text":{"type":"string","description":"Text to insert at the focused element's cursor."},+ "window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used."},+ "element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."},+ "delay_ms":{"type":"integer","minimum":0,"maximum":200,"description":"Milliseconds between characters. Default 30."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(p) => p,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let pid = raw_pid as u32;+ let text = match args.get("text").and_then(|v| v.as_str()) {+ Some(t) => t.to_owned(),+ None => return ToolResult::error("Missing required string field text."),+ };+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ let elem_idx = args.get("element_index").and_then(|v| v.as_u64());+ if elem_idx.is_some() && hwnd_opt.is_none() {+ return ToolResult::error(+ "window_id is required when element_index is used — the element_index cache \+ is scoped per (pid, window_id). Pass the same window_id you used in \+ `get_window_state`.");+ }+ let _delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30);+ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };+ let text_len = text.chars().count();+ let result = tokio::task::spawn_blocking(move || {+ crate::input::post_type_text(hwnd, &text)+ }).await;+ match result {+ // Match Swift's CGEvent-fallback text format 1:1 (Windows always+ // takes the synthesis path since AXSelectedText has no UIA+ // equivalent wired up here yet).+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Typed {text_len} char(s) on pid {raw_pid} via PostMessage ({_delay_ms}ms delay)."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── press_key ─────────────────────────────────────────────────────────────────++pub struct PressKeyTool;+static PRESS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for PressKeyTool {+ fn def(&self) -> &ToolDef {+ PRESS_DEF.get_or_init(|| ToolDef {+ name: "press_key".into(),+ // Description ported from Swift `PressKeyTool.swift` with+ // Windows-specific transport note (PostMessage WM_KEYDOWN/UP).+ description: "Press and release a single key, delivered directly to the target pid's \+ top-level window via PostMessage(WM_KEYDOWN/WM_KEYUP). The target does NOT need \+ to be frontmost — no focus steal.\n\n\+ Optional `window_id` selects a specific HWND when the pid owns more than one; \+ without it the first visible top-level window for the pid is used.\n\n\+ Key vocabulary: return, tab, escape, up/down/left/right, space, delete, home, \+ end, pageup, pagedown, f1-f12, plus any letter or digit. Optional `modifiers` \+ array takes ctrl/shift/alt/win. For true combinations (ctrl+c), `hotkey` is a \+ cleaner surface.\n\n\+ Note: `element_index` is accepted for cross-platform parity with macOS but \+ currently no-op on Windows (no UIA SetFocus path wired up yet).".into(),+ input_schema: json!({+ "type":"object","required":["pid","key"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "key":{"type":"string","description":"Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit)."},+ "modifiers":{"type":"array","items":{"type":"string"},"description":"Optional modifier names held while the key is pressed (ctrl/shift/alt/win)."},+ "element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."},+ "window_id":{"type":"integer","description":"HWND for the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(p) => p,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let pid = raw_pid as u32;+ let key = match args.get("key").and_then(|v| v.as_str()) {+ Some(k) => k.to_owned(),+ None => return ToolResult::error("Missing required string field key."),+ };+ let mods: Vec<String> = args.get("modifiers")+ .and_then(|v| v.as_array())+ .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ // Swift requires window_id when element_index is supplied — ports the+ // same validation.+ let elem_idx = args.get("element_index").and_then(|v| v.as_u64());+ if elem_idx.is_some() && hwnd_opt.is_none() {+ return ToolResult::error(+ "window_id is required when element_index is used — the element_index cache \+ is scoped per (pid, window_id). Pass the same window_id you used in \+ `get_window_state`.");+ }+ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };+ let key_display = key.clone();+ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = mods.iter().map(String::as_str).collect();+ crate::input::post_key(hwnd, &key, &m)+ }).await;+ match result {+ // Match Swift's text format 1:1: `"✅ Pressed KEY on pid X."`.+ Ok(Ok(())) => ToolResult::text(format!("✅ Pressed {key_display} on pid {raw_pid}.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── hotkey ────────────────────────────────────────────────────────────────────++fn is_modifier(k: &str) -> bool {+ matches!(k.to_lowercase().as_str(),+ "ctrl" | "control" | "shift" | "alt" | "win" | "windows" | "cmd" | "command")+}++pub struct HotkeyTool;+static HOTKEY_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for HotkeyTool {+ fn def(&self) -> &ToolDef {+ HOTKEY_DEF.get_or_init(|| ToolDef {+ name: "hotkey".into(),+ // Description ported from Swift `HotkeyTool.swift` (omitting macOS-+ // specific FocusWithoutRaise/SLEventPostToPid mechanics — Windows+ // uses PostMessage which doesn't need a NSMenu-activation dance).+ description: "Press a combination of keys simultaneously — e.g. `[\"ctrl\", \"c\"]` \+ for Copy, `[\"ctrl\", \"shift\", \"t\"]` for reopen-closed-tab. The combo is \+ posted directly to the target pid's window via PostMessage(WM_KEYDOWN/UP); \+ the target does NOT need to be frontmost.\n\n\+ **`window_id`** (optional): explicit HWND when the pid owns more than one \+ window; otherwise the pid's first visible window is used.\n\n\+ Recognized modifiers: ctrl/control, shift, alt, win/windows. Non-modifier \+ keys use the same vocabulary as `press_key` (return, tab, escape, \+ up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, \+ letters, digits). Order: modifiers first, one non-modifier last.".into(),+ input_schema: json!({+ "type":"object","required":["pid","keys"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "keys":{"type":"array","items":{"type":"string"},"minItems":2,+ "description":"Modifier(s) and one non-modifier key, e.g. [\"ctrl\", \"c\"]."},+ "window_id":{"type":"integer","description":"Explicit HWND when the pid owns multiple windows."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ // Swift error wording 1:1.+ let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(p) => p,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let pid = raw_pid as u32;++ // Parse keys array (Swift's only path). Legacy key+modifiers shape+ // still accepted as a Rust-side convenience.+ let (key, mods, full_keys) = if let Some(arr) = args.get("keys") {+ let raw = match arr.as_array() {+ Some(r) => r,+ None => return ToolResult::error("Missing required array field keys."),+ };+ let keys: Vec<String> = raw.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect();+ if keys.len() != raw.len() || keys.is_empty() {+ return ToolResult::error("keys must be a non-empty array of strings.");+ }+ let modifiers: Vec<String> = keys.iter().filter(|k| is_modifier(k)).cloned().collect();+ let non_mods: Vec<String> = keys.iter().filter(|k| !is_modifier(k)).cloned().collect();+ if non_mods.is_empty() {+ return ToolResult::error("keys must include at least one non-modifier key.");+ }+ (non_mods.last().unwrap().clone(), modifiers, keys)+ } else if let Some(k) = args.get("key").and_then(|v| v.as_str()) {+ let m: Vec<String> = args.get("modifiers").and_then(|v| v.as_array())+ .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())+ .unwrap_or_default();+ let mut full = m.clone(); full.push(k.to_owned());+ (k.to_owned(), m, full)+ } else {+ return ToolResult::error("Missing required array field keys.");+ };++ // Resolve HWND: use window_id if given, else pick first window for pid.+ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let pid2 = pid;+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid2))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };++ // Match Swift's text format exactly: `"✅ Pressed K1+K2+... on pid X."`+ // where K1+K2+... is `keys.joined(separator: "+")` of the FULL keys+ // array as the caller supplied it (modifiers + non-modifier in order).+ let key_display = full_keys.join("+");+ let result = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = mods.iter().map(String::as_str).collect();+ crate::input::post_key(hwnd, &key, &m)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("✅ Pressed {key_display} on pid {raw_pid}.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── set_value ─────────────────────────────────────────────────────────────────++pub struct SetValueTool {+ state: Arc<ToolState>,+}++static SET_VALUE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetValueTool {+ fn def(&self) -> &ToolDef {+ SET_VALUE_DEF.get_or_init(|| ToolDef {+ name: "set_value".into(),+ // Description ported from Swift `SetValueTool.swift` with the+ // Windows transport (UIA ValuePattern instead of AXValue).+ // Swift's AXPopUpButton special-case has a UIA analogue+ // (SelectionPattern) that's not yet wired up here; documented.+ description: "Set a value on a UIA element via the ValuePattern interface.\n\n\+ Two semantic modes (matching Swift's split):\n\+ - **Standard input** (text fields, sliders, combo box edit): writes the value \+ directly through UIA `IUIAutomationValuePattern::SetValue`. This is the \+ canonical Windows write path, equivalent to Swift's `AXValue` write.\n\+ - **ComboBox / select dropdown**: ValuePattern.SetValue picks the option \+ whose text matches `value` on most native ComboBox controls.\n\n\+ For free-form text entry that the target accepts via keystrokes only, prefer \+ `type_text` — UIA ValuePattern writes are ignored by some web inputs (same \+ caveat as Swift's `AXValue`-vs-WebKit).".into(),+ input_schema: json!({+ "type":"object","required":["pid","window_id","element_index","value"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "window_id":{"type":"integer","description":"HWND of the window. The element_index cache is scoped per (pid, window_id)."},+ "element_index":{"type":"integer","description":"Element index from the last get_window_state."},+ "value":{"type":"string","description":"New value. UIA will coerce to the element's native type."}+ },"additionalProperties":false+ }),+ // Swift: idempotent: true (setting same value twice is idempotent).+ read_only: false, destructive: true, idempotent: true, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift's "Missing required integer fields pid, window_id, and element_index."+ let mut missing_ints: Vec<&str> = Vec::new();+ let raw_pid = args.get("pid").and_then(|v| v.as_i64());+ if raw_pid.is_none() { missing_ints.push("pid"); }+ if args.get("window_id").and_then(|v| v.as_u64()).is_none() { missing_ints.push("window_id"); }+ if args.get("element_index").and_then(|v| v.as_u64()).is_none() { missing_ints.push("element_index"); }+ if !missing_ints.is_empty() {+ // Swift wording when all three are missing; for partial misses+ // we still emit the same shape (good enough for parity).+ return ToolResult::error("Missing required integer fields pid, window_id, and element_index.");+ }+ let pid = raw_pid.unwrap() as u32;+ let hwnd = args.get("window_id").and_then(|v| v.as_u64()).unwrap();+ let idx = args.get("element_index").and_then(|v| v.as_u64()).unwrap() as usize;+ let value = match args.get("value").and_then(|v| v.as_str()) {+ Some(v) => v.to_owned(),+ None => return ToolResult::error("Missing required string field value."),+ };++ let state = self.state.clone();+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {+ let ptr = state.element_cache.get_element_ptr(pid, hwnd, idx)+ .ok_or_else(|| anyhow::anyhow!("Element {idx} not in cache."))?;+ use windows::Win32::UI::Accessibility::{IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId};+ use windows::core::{Interface, BSTR};+ let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };+ let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId)? };+ std::mem::forget(elem);+ let vp: IUIAutomationValuePattern = pattern.cast()?;+ unsafe { vp.SetValue(&BSTR::from(value.as_str()))? };+ Ok(())+ }).await;+ match result {+ // Match Swift's text format 1:1 (default AXValue-write path).+ // UIA role/title placeholder pending element-cache enrichment.+ Ok(Ok(())) => ToolResult::text(format!("✅ Set AXValue on [{idx}] (UIA ValuePattern).")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── scroll ────────────────────────────────────────────────────────────────────++pub struct ScrollTool;+static SCROLL_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ScrollTool {+ fn def(&self) -> &ToolDef {+ SCROLL_DEF.get_or_init(|| ToolDef {+ name: "scroll".into(),+ // Description ported from Swift `ScrollTool.swift` with the+ // Windows-specific transport note (WM_VSCROLL/WM_HSCROLL).+ description: "Scroll the target pid's focused region.\n\n\+ Windows transport: WM_VSCROLL / WM_HSCROLL posted to the window — the same \+ events the OS sends when scrollbars or trackpads scroll, so any window that \+ handles scrollbars correctly responds to this. Swift uses synthesized \+ keystrokes (PageDown / arrow keys) via auth-signed SLEventPostToPid; both \+ approaches reach backgrounded windows.\n\n\+ Mapping: `by: \"page\"` → SB_PAGEDOWN/UP/LEFT/RIGHT × amount; \+ `by: \"line\"` → SB_LINEDOWN/UP/LEFT/RIGHT × amount.\n\n\+ Note: `element_index` is accepted for cross-platform parity but currently \+ no-op on Windows (UIA SetFocus not wired up yet — same caveat as `press_key`).".into(),+ input_schema: json!({+ "type":"object","required":["pid","direction"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "direction":{"type":"string","enum":["up","down","left","right"]},+ "by":{"type":"string","enum":["line","page"],"description":"Scroll granularity. Default: line."},+ "amount":{"type":"integer","minimum":1,"maximum":50,+ "description":"Number of scroll ticks. Default 3."},+ "window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."},+ "element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: false, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(p) => p,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let pid = raw_pid as u32;+ let direction = match args.get("direction").and_then(|v| v.as_str()) {+ Some(d) => d.to_owned(),+ None => return ToolResult::error("Missing required string field direction."),+ };+ let by = args.get("by").and_then(|v| v.as_str()).unwrap_or("line").to_owned();+ let direction_display = direction.clone();+ let by_display = by.clone();+ let amount = args.get("amount").and_then(|v| v.as_u64())+ .unwrap_or(3).clamp(1, 50) as u32;+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ let elem_idx = args.get("element_index").and_then(|v| v.as_u64());+ if elem_idx.is_some() && hwnd_opt.is_none() {+ return ToolResult::error(+ "window_id is required when element_index is used — the element_index cache \+ is scoped per (pid, window_id). Pass the same window_id you used in \+ `get_window_state`.");+ }++ // Resolve HWND: use window_id if given, else first window for pid.+ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };++ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {+ use windows::Win32::Foundation::{HWND, LPARAM, WPARAM};+ use windows::Win32::UI::WindowsAndMessaging::{+ PostMessageW, WM_HSCROLL, WM_VSCROLL,+ SB_LINEDOWN, SB_LINEUP, SB_LINELEFT, SB_LINERIGHT,+ SB_PAGEDOWN, SB_PAGEUP, SB_PAGELEFT, SB_PAGERIGHT,+ };++ let hwnd_win = HWND(hwnd as *mut _);+ let use_page = by == "page";+ let (msg, code) = match (direction.as_str(), use_page) {+ ("up", false) => (WM_VSCROLL, SB_LINEUP),+ ("up", true) => (WM_VSCROLL, SB_PAGEUP),+ ("down", false) => (WM_VSCROLL, SB_LINEDOWN),+ ("down", true) => (WM_VSCROLL, SB_PAGEDOWN),+ ("left", false) => (WM_HSCROLL, SB_LINELEFT),+ ("left", true) => (WM_HSCROLL, SB_PAGELEFT),+ ("right", false) => (WM_HSCROLL, SB_LINERIGHT),+ ("right", true) => (WM_HSCROLL, SB_PAGERIGHT),+ _ => (WM_VSCROLL, SB_LINEDOWN),+ };+ for _ in 0..amount {+ unsafe {+ PostMessageW(hwnd_win, msg, WPARAM(code.0 as usize), LPARAM(0))?;+ }+ }+ Ok(())+ }).await;++ match result {+ Ok(Ok(())) => {+ // Match Swift's text format shape 1:1 (Swift uses key names+ // like "pagedown"; Windows uses Win32 SB_* constants — show+ // the actual mechanism for traceability).+ let mech_name = match (direction_display.as_str(), by_display.as_str()) {+ ("up", "page") => "SB_PAGEUP",+ ("down", "page") => "SB_PAGEDOWN",+ ("left", "page") => "SB_PAGELEFT",+ ("right", "page") => "SB_PAGERIGHT",+ ("up", _) => "SB_LINEUP",+ ("down", _) => "SB_LINEDOWN",+ ("left", _) => "SB_LINELEFT",+ ("right", _) => "SB_LINERIGHT",+ _ => "SB_LINEDOWN",+ };+ ToolResult::text(format!(+ "✅ Scrolled pid {raw_pid} {direction_display} via {amount}× {mech_name} message(s)."+ ))+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── screenshot ────────────────────────────────────────────────────────────────++pub struct ScreenshotTool {+ state: Arc<ToolState>,+}+static SCREENSHOT_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ScreenshotTool {+ fn def(&self) -> &ToolDef {+ SCREENSHOT_DEF.get_or_init(|| ToolDef {+ name: "screenshot".into(),+ // Description ported from Swift `ScreenshotTool.swift` with the+ // Windows-specific transport note (BitBlt + PrintWindow instead of+ // ScreenCaptureKit).+ description: "Capture a screenshot of a single window via BitBlt + PrintWindow \+ (no focus change). Returns base64-encoded image data in the requested format \+ (default png).\n\n\+ `window_id` is recommended (use `list_windows` to find it). When omitted, \+ captures the full primary display — a Windows-only convenience for whole-\+ screen snapshots without a per-window target.\n\n\+ Requires no special permissions on Windows.".into(),+ input_schema: json!({+ "type":"object","properties":{+ "window_id":{"type":"integer","description":"HWND of the window to capture. When omitted, captures the full primary display (Windows-only)."},+ "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: png."},+ "quality":{"type":"integer","minimum":1,"maximum":95,+ "description":"JPEG quality 1-95; ignored for png."}+ },"additionalProperties":false+ }),+ read_only: true, destructive: false, idempotent: false, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();+ let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8;+ let is_jpeg = format == "jpeg";+ let max_dim = self.state.config.read().unwrap().max_image_dimension;++ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(String, u32, u32)> {+ let raw = match hwnd_opt {+ Some(hwnd) => crate::capture::screenshot_window_bytes(hwnd)?,+ None => crate::capture::screenshot_display_bytes()?,+ };+ let png_bytes = crate::capture::resize_png_if_needed(&raw, max_dim)?;+ let (w, h) = crate::capture::png_dimensions_pub(&png_bytes)?;+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};+ if is_jpeg {+ let jpeg = crate::capture::png_bytes_to_jpeg(&png_bytes, quality)?;+ Ok((B64.encode(&jpeg), w, h))+ } else {+ Ok((B64.encode(&png_bytes), w, h))+ }+ }).await;++ match result {+ Ok(Ok((b64, w, h))) => {+ let label = if is_jpeg { "jpeg" } else { "png" };+ // Match Swift `ScreenshotTool.swift` text format 1:1:+ // "✅ Window screenshot — WxH png [window_id: ID]"+ // Whole-display fallback uses "✅ Display screenshot — WxH png"+ // (Rust-only path; Swift always requires window_id).+ let summary = match hwnd_opt {+ Some(wid) => format!("✅ Window screenshot — {w}x{h} {label} [window_id: {wid}]"),+ None => format!("✅ Display screenshot — {w}x{h} {label}"),+ };+ let mut tr = ToolResult::text(summary);+ let img_content = if is_jpeg {+ mcp_server::protocol::Content::image_jpeg(b64)+ } else {+ mcp_server::protocol::Content::image_png(b64)+ };+ tr.content.push(img_content);+ tr.structured_content = Some(json!({ "width": w, "height": h, "format": label }));+ tr+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── double_click ──────────────────────────────────────────────────────────────++pub struct DoubleClickTool {+ state: Arc<ToolState>,+}++static DCLICK_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for DoubleClickTool {+ fn def(&self) -> &ToolDef {+ DCLICK_DEF.get_or_init(|| ToolDef {+ name: "double_click".into(),+ // Description ported from Swift `DoubleClickTool.swift` semantics+ // (two addressing modes, AXOpen analogue on Windows is not yet+ // wired up — element path always falls back to a stamped pixel+ // double-click via PostMessage).+ description: "Double-click against a target pid. Two addressing modes:\n\n\+ - `element_index` + `window_id` (from the last `get_window_state` snapshot \+ of that window) — synthesizes a stamped pixel double-click at the element's \+ cached on-screen center via PostMessage. (Windows has no AXOpen analogue; \+ Swift's AXOpen-first path falls through to the same pixel recipe when the \+ element doesn't advertise AXOpen, so the user-visible behavior matches.)\n\n\+ - `x`, `y` (window-local screenshot pixels, top-left origin of the PNG \+ returned by `get_window_state`) — posts two WM_LBUTTONDOWN/UP pairs in \+ quick succession to the deepest child window at that point.\n\n\+ Exactly one of `element_index` or (`x` AND `y`) must be provided. \+ `pid` is required in both modes. `window_id` is required when \+ `element_index` is used. The macOS-only `modifier` field is accepted for \+ parity (no-op on Windows — PostMessage doesn't propagate modifier-key state).".into(),+ input_schema: json!({"type":"object","required":["pid"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "window_id":{"type":"integer","description":"HWND for the target window. Required when element_index is used."},+ "element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."},+ "x":{"type":"number","description":"X in window-local screenshot pixels. Must be provided together with y."},+ "y":{"type":"number","description":"Y in window-local screenshot pixels. Must be provided together with x."},+ "modifier":{"type":"array","items":{"type":"string"},"description":"Modifier keys held during the gesture. Accepted for parity; currently no-op on Windows."},+ "from_zoom":{"type":"boolean","description":"After a zoom call, pass true to translate zoom-image coords back to window space."}+ },"additionalProperties":false}),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(p) => p,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let pid = raw_pid as u32;+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ let elem_idx = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let x = args.get("x").and_then(|v| v.as_f64());+ let y = args.get("y").and_then(|v| v.as_f64());+ // Swift validates "both x and y or neither" and "no element_index without window_id".+ let has_xy = x.is_some() && y.is_some();+ let partial_xy = x.is_some() != y.is_some();+ if partial_xy {+ return ToolResult::error("Provide both x and y together, not just one.");+ }+ if elem_idx.is_some() && has_xy {+ return ToolResult::error("Provide either element_index or (x, y), not both.");+ }+ if elem_idx.is_none() && !has_xy {+ return ToolResult::error("Provide element_index or (x, y) to address the double-click target.");+ }+ if elem_idx.is_some() && hwnd_opt.is_none() {+ return ToolResult::error(+ "window_id is required when element_index is used — the element_index cache \+ is scoped per (pid, window_id). Pass the same window_id you used in \+ `get_window_state`.");+ }++ // Resolve HWND: explicit, or auto from pid.+ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };++ if let Some(idx) = elem_idx {+ let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {+ Some(v) => v,+ None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")),+ };+ overlay_glide_to(cx as f64, cy as f64).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 });+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {+ crate::input::post_click_screen(hwnd, cx, cy, 2, "left")?;+ // Swift text format 1:1: `"✅ Posted double-click to [N] role \"title\" at screen-point (X, Y)."`.+ // UIA role/title placeholder pending element-cache enrichment.+ Ok(format!("✅ Posted double-click to [{idx}] at screen-point ({cx}, {cy})."))+ }).await;+ match result {+ Ok(Ok(msg)) => ToolResult::text(msg),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ } else if let (Some(mut px), Some(mut py)) = (x, y) {+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);+ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(px, py); px = wx; py = wy; }+ None => return ToolResult::error(+ format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")+ ),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ px *= ratio;+ py *= ratio;+ }+ let (sx, sy) = unsafe {+ use windows::Win32::Foundation::{HWND, POINT};+ use windows::Win32::Graphics::Gdi::ClientToScreen;+ let mut pt = POINT { x: px as i32, y: py as i32 };+ let _ = ClientToScreen(HWND(hwnd as *mut _), &mut pt);+ (pt.x as f64, pt.y as f64)+ };+ overlay_glide_to(sx, sy).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ let (xi, yi) = (px as i32, py as i32);+ let result = tokio::task::spawn_blocking(move || crate::input::post_click(hwnd, xi, yi, 2, "left")).await;+ match result {+ Ok(Ok(())) => {+ // Match Swift's pixel-path text 1:1.+ ToolResult::text(format!(+ "✅ Posted double-click to pid {pid} at window-pixel ({xi}, {yi}) → screen-point ({}, {}).",+ sx as i32, sy as i32))+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ } else {+ // Unreachable — guarded by the earlier validation block.+ unreachable!()+ }+ }+}++// ── right_click ───────────────────────────────────────────────────────────────++pub struct RightClickTool {+ state: Arc<ToolState>,+}++static RCLICK_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for RightClickTool {+ fn def(&self) -> &ToolDef {+ RCLICK_DEF.get_or_init(|| ToolDef {+ name: "right_click".into(),+ // Description ported from Swift `RightClickTool.swift` semantics.+ // Windows uses PostMessage(WM_RBUTTONDOWN/UP); the AXShowMenu+ // analogue (UIA ShowContextMenu) is not yet wired up, so element+ // path falls through to the pixel recipe at the cached center.+ description: "Right-click against a target pid. Two addressing modes:\n\n\+ - `element_index` + `window_id` (from the last `get_window_state` snapshot \+ of that window) — posts WM_RBUTTONDOWN/UP at the element's cached on-screen \+ center via PostMessage. (Windows has no AXShowMenu analogue wired up yet; \+ Swift's AX-action path falls through to the same pixel recipe on non-\+ advertising elements, so user-visible behavior matches.)\n\n\+ - `x`, `y` (window-local screenshot pixels, top-left origin of the PNG \+ returned by `get_window_state`) — posts WM_RBUTTONDOWN/UP to the deepest \+ child window at that point.\n\n\+ Exactly one of `element_index` or (`x` AND `y`) must be provided. `pid` is \+ required in both modes. `window_id` is required when `element_index` is used. \+ `modifier` is accepted for parity (no-op on Windows — PostMessage doesn't \+ propagate modifier-key state).".into(),+ input_schema: json!({"type":"object","required":["pid"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "window_id":{"type":"integer","description":"HWND for the target window. Required when element_index is used."},+ "element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."},+ "x":{"type":"number","description":"X in window-local screenshot pixels. Must be provided together with y."},+ "y":{"type":"number","description":"Y in window-local screenshot pixels. Must be provided together with x."},+ "modifier":{"type":"array","items":{"type":"string"},"description":"Modifier keys held during the right-click. Accepted for parity; currently no-op on Windows."},+ "from_zoom":{"type":"boolean","description":"After a zoom call, pass true to translate zoom-image coords back to window space."}+ },"additionalProperties":false}),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(p) => p,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let pid = raw_pid as u32;+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ let elem_idx = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize);+ let x = args.get("x").and_then(|v| v.as_f64());+ let y = args.get("y").and_then(|v| v.as_f64());+ // Port Swift's full validation set.+ let has_xy = x.is_some() && y.is_some();+ let partial_xy = x.is_some() != y.is_some();+ if partial_xy {+ return ToolResult::error("Provide both x and y together, not just one.");+ }+ if elem_idx.is_some() && has_xy {+ return ToolResult::error("Provide either element_index or (x, y), not both.");+ }+ if elem_idx.is_none() && !has_xy {+ return ToolResult::error("Provide element_index or (x, y) to address the right-click target.");+ }+ if elem_idx.is_some() && hwnd_opt.is_none() {+ return ToolResult::error(+ "window_id is required when element_index is used — the element_index cache \+ is scoped per (pid, window_id). Pass the same window_id you used in \+ `get_window_state`.");+ }++ // Resolve HWND: explicit, or auto from pid.+ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };++ if let Some(idx) = elem_idx {+ let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {+ Some(v) => v,+ None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")),+ };+ overlay_glide_to(cx as f64, cy as f64).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 });+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {+ crate::input::post_click_screen(hwnd, cx, cy, 1, "right")?;+ // Match Swift's element-path text 1:1+ // (`"✅ Shown menu for [N] role \"title\"."`). UIA role/title+ // placeholder pending element-cache enrichment.+ Ok(format!("✅ Shown menu for [{idx}] (screen ({cx}, {cy}))."))+ }).await;+ match result {+ Ok(Ok(msg)) => ToolResult::text(msg),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ } else if let (Some(mut px), Some(mut py)) = (x, y) {+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);+ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(px, py); px = wx; py = wy; }+ None => return ToolResult::error(+ format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")+ ),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ px *= ratio;+ py *= ratio;+ }+ let (sx, sy) = unsafe {+ use windows::Win32::Foundation::{HWND, POINT};+ use windows::Win32::Graphics::Gdi::ClientToScreen;+ let mut pt = POINT { x: px as i32, y: py as i32 };+ let _ = ClientToScreen(HWND(hwnd as *mut _), &mut pt);+ (pt.x as f64, pt.y as f64)+ };+ overlay_glide_to(sx, sy).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ let (xi, yi) = (px as i32, py as i32);+ let result = tokio::task::spawn_blocking(move || crate::input::post_click(hwnd, xi, yi, 1, "right")).await;+ match result {+ Ok(Ok(())) => {+ // Swift pixel-path text 1:1.+ ToolResult::text(format!(+ "✅ Posted right-click to pid {pid} at window-pixel ({xi}, {yi}) → screen-point ({}, {}).",+ sx as i32, sy as i32))+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ } else {+ // Guarded above by the early validation block — unreachable.+ unreachable!()+ }+ }+}++// ── drag ─────────────────────────────────────────────────────────────────────++pub struct DragTool {+ state: Arc<ToolState>,+}++static DRAG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for DragTool {+ fn def(&self) -> &ToolDef {+ DRAG_DEF.get_or_init(|| ToolDef {+ name: "drag".into(),+ description: "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. \+ duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate \+ WM_MOUSEMOVE events along the path. No focus steal. After a zoom call, pass from_zoom=true.".into(),+ input_schema: json!({"type":"object","required":["pid","from_x","from_y","to_x","to_y"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "window_id":{"type":"integer","description":"Target window handle (HWND). Optional — driver picks frontmost window of pid when omitted."},+ "from_x":{"type":"number","description":"Drag-start X in window-local screenshot pixels."},+ "from_y":{"type":"number","description":"Drag-start Y in window-local screenshot pixels."},+ "to_x":{"type":"number","description":"Drag-end X in window-local screenshot pixels."},+ "to_y":{"type":"number","description":"Drag-end Y in window-local screenshot pixels."},+ "duration_ms":{"type":"integer","minimum":0,"maximum":10000,"description":"Wall-clock duration of drag path. Default: 500."},+ "steps":{"type":"integer","minimum":1,"maximum":200,"description":"Number of intermediate WM_MOUSEMOVE events. Default: 20."},+ "modifier":{"type":"array","items":{"type":"string"},"description":"Modifier keys: cmd/shift/option/ctrl (held via keyboard)."},+ "button":{"type":"string","enum":["left","right","middle"],"description":"Mouse button. Default: left."},+ "from_zoom":{"type":"boolean","description":"When true, coordinates are in the last zoom image for this pid."}+ },"additionalProperties":false}),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(p) => p,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let pid = raw_pid as u32;++ let coerce = |key: &str| -> Option<f64> {+ args.get(key).and_then(|v| v.as_f64())+ .or_else(|| args.get(key).and_then(|v| v.as_i64()).map(|i| i as f64))+ };+ let from_x_opt = coerce("from_x");+ let from_y_opt = coerce("from_y");+ let to_x_opt = coerce("to_x");+ let to_y_opt = coerce("to_y");+ let (mut from_x, mut from_y, mut to_x, mut to_y) =+ match (from_x_opt, from_y_opt, to_x_opt, to_y_opt) {+ (Some(a), Some(b), Some(c), Some(d)) => (a, b, c, d),+ _ => return ToolResult::error(+ "from_x, from_y, to_x, and to_y are all required (window-local pixels)."),+ };++ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ let duration_ms = args.get("duration_ms").and_then(|v| v.as_u64()).unwrap_or(500);+ let steps = args.get("steps").and_then(|v| v.as_u64()).unwrap_or(20) as usize;+ let button = args.get("button").and_then(|v| v.as_str()).unwrap_or("left").to_owned();+ let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);++ if from_zoom {+ match self.state.zoom_registry.get(pid) {+ Some(ctx) => {+ let (wx, wy) = ctx.zoom_to_window(from_x, from_y);+ let (wx2, wy2) = ctx.zoom_to_window(to_x, to_y);+ from_x = wx; from_y = wy; to_x = wx2; to_y = wy2;+ }+ None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")),+ }+ } else if let Some(ratio) = self.state.resize_registry.ratio(pid) {+ from_x *= ratio; from_y *= ratio;+ to_x *= ratio; to_y *= ratio;+ }++ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };++ // Compute screen-coord endpoints for Swift's text format.+ let (sx_from, sy_from, sx_to, sy_to) = unsafe {+ use windows::Win32::Foundation::{HWND, POINT};+ use windows::Win32::Graphics::Gdi::ClientToScreen;+ let mut p1 = POINT { x: from_x as i32, y: from_y as i32 };+ let mut p2 = POINT { x: to_x as i32, y: to_y as i32 };+ let _ = ClientToScreen(HWND(hwnd as *mut _), &mut p1);+ let _ = ClientToScreen(HWND(hwnd as *mut _), &mut p2);+ (p1.x, p1.y, p2.x, p2.y)+ };++ let button_c = button.clone();+ let result = tokio::task::spawn_blocking(move || {+ crate::input::mouse::post_drag(+ hwnd,+ from_x as i32, from_y as i32,+ to_x as i32, to_y as i32,+ duration_ms, steps, &button_c,+ )+ }).await;++ match result {+ Ok(Ok(())) => {+ // Match Swift's text format verbatim.+ let button_suffix = if button == "left" { String::new() } else { format!(" ({} button)", button) };+ ToolResult::text(format!(+ "✅ Posted drag{button_suffix} to pid {raw_pid} \+ from window-pixel ({}, {}) → ({}, {}), \+ screen ({sx_from}, {sy_from}) → ({sx_to}, {sy_to}) \+ in {duration_ms}ms / {steps} steps.",+ from_x as i32, from_y as i32, to_x as i32, to_y as i32,+ ))+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── get_screen_size ───────────────────────────────────────────────────────────++pub struct GetScreenSizeTool;+static GSS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetScreenSizeTool {+ fn def(&self) -> &ToolDef {+ GSS_DEF.get_or_init(|| ToolDef {+ name: "get_screen_size".into(),+ description: "Return the logical size of the main display in points plus its backing \+ scale factor. Agents click in points; Retina displays have scale_factor 2.0. \+ Requires no TCC permissions.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};+ use windows::Win32::UI::HiDpi::GetDpiForSystem;+ // SM_CXSCREEN/SM_CYSCREEN return values in the DPI of the current+ // process; for a DPI-unaware process they're in *logical* points,+ // matching Swift's NSScreen.frame. Scale = system DPI / 96.+ let (w, h) = unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) };+ let dpi = unsafe { GetDpiForSystem() };+ let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 };+ // Matches Swift text format 1:1.+ ToolResult::text(format!("✅ Main display: {w}x{h} points @ {scale}x"))+ .with_structured(json!({ "width": w, "height": h, "scale_factor": scale }))+ }+}++// ── get_cursor_position ───────────────────────────────────────────────────────++pub struct GetCursorPositionTool;+static GCP_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetCursorPositionTool {+ fn def(&self) -> &ToolDef {+ GCP_DEF.get_or_init(|| ToolDef {+ name: "get_cursor_position".into(),+ description: "Return the current mouse cursor position in screen points (origin top-left).".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ use windows::Win32::Foundation::POINT;+ use windows::Win32::UI::WindowsAndMessaging::GetCursorPos;+ let mut pt = POINT { x: 0, y: 0 };+ unsafe { let _ = GetCursorPos(&mut pt); }+ // Text format matches Swift `GetCursorPositionTool` 1:1.+ ToolResult::text(format!("✅ Cursor at ({}, {})", pt.x, pt.y))+ .with_structured(json!({ "x": pt.x, "y": pt.y }))+ }+}++// ── move_cursor ───────────────────────────────────────────────────────────────++pub struct MoveCursorTool {+ state: Arc<ToolState>,+}++static MCURSOR_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for MoveCursorTool {+ fn def(&self) -> &ToolDef {+ MCURSOR_DEF.get_or_init(|| ToolDef {+ name: "move_cursor".into(),+ description: "Move the agent cursor overlay to (x, y). Does NOT move the real mouse cursor.".into(),+ input_schema: json!({"type":"object","required":["x","y"],"properties":{+ "x":{"type":"number"},"y":{"type":"number"},"cursor_id":{"type":"string"}+ },"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ self.state.cursor_registry.update_position(cursor_id, x, y);+ // End pointing upper-left (45°) — matches Swift's+ // `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention so+ // the cursor settles to the natural macOS-style pose.+ crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo {+ x, y, end_heading_radians: std::f64::consts::FRAC_PI_4,+ });+ ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1})."))+ }+}++// ── set_agent_cursor_enabled ──────────────────────────────────────────────────++pub struct SetAgentCursorEnabledTool {+ state: Arc<ToolState>,+}++static SCE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetAgentCursorEnabledTool {+ fn def(&self) -> &ToolDef {+ SCE_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_enabled".into(),+ // Description ported from Swift `SetAgentCursorEnabledTool.swift`.+ description: "Toggle the visual agent-cursor overlay. When enabled, future \+ pointer actions animate a floating arrow to the target's on-screen position \+ before firing the click — purely visual, the click dispatch itself is \+ unchanged. Disabling removes the overlay immediately.\n\n\+ Default: enabled. Stays on for the life of the MCP session / daemon; \+ disable with `{\"enabled\": false}` for headless / CI runs where the \+ visual isn't wanted.\n\n\+ Rust-only: `cursor_id` selects an instance from the multi-cursor registry; \+ default is `'default'` (Swift has a single AgentCursor.shared).".into(),+ input_schema: json!({"type":"object","required":["enabled"],"properties":{+ "enabled":{"type":"boolean","description":"True to show the overlay cursor; false to hide."},+ "cursor_id":{"type":"string","description":"Rust-only: multi-cursor instance id. Default 'default'."}+ },"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ // Swift error wording 1:1.+ let enabled = match args.get("enabled").and_then(|v| v.as_bool()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required boolean field `enabled`."),+ };+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ self.state.cursor_registry.set_enabled(cursor_id, enabled);+ crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled));+ // Match Swift text format 1:1: `"✅ Agent cursor enabled."`+ // (or `"✅ Agent cursor disabled."`).+ ToolResult::text(if enabled {+ "✅ Agent cursor enabled.".to_owned()+ } else {+ "✅ Agent cursor disabled.".to_owned()+ })+ }+}++// ── set_agent_cursor_motion ───────────────────────────────────────────────────++pub struct SetAgentCursorMotionTool {+ state: Arc<ToolState>,+}++static CURSOR_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetAgentCursorMotionTool {+ fn def(&self) -> &ToolDef {+ CURSOR_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_motion".into(),+ description: "Configure the visual appearance and motion curve of an agent cursor instance.\n\n\+ Appearance:\n\+ - cursor_id: instance name (default='default')\n\+ - cursor_icon: built-in ('arrow','crosshair','hand','dot') or PNG/SVG file path\n\+ - cursor_color: hex color e.g. '#00FFFF' or CSS name\n\+ - cursor_label: short text shown near the cursor\n\+ - cursor_size: dot radius in points (default=16)\n\+ - cursor_opacity: 0.0–1.0 (default=0.85)\n\n\+ Motion curve (Bezier):\n\+ - arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n\+ - spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n\+ - glide_duration_ms: flight duration per move [50,5000]. Default 160\n\+ - dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n\+ - idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000".into(),+ input_schema: json!({+ "type":"object","properties":{+ "cursor_id":{"type":"string"},+ "cursor_icon":{"type":"string"},+ "cursor_color":{"type":"string"},+ "cursor_label":{"type":"string"},+ "cursor_size":{"type":"number"},+ "cursor_opacity":{"type":"number"},+ "start_handle":{"type":"number","description":"Start-handle fraction [0,1]. Default 0.3."},+ "end_handle":{"type":"number","description":"End-handle fraction [0,1]. Default 0.3."},+ "arc_size":{"type":"number","description":"Arc deflection as fraction of path length [0,1]. Default 0.25."},+ "arc_flow":{"type":"number","description":"Asymmetry bias [-1,1]. Default 0.0."},+ "spring":{"type":"number","description":"Settle damping [0.3,1.0]. Default 0.72."},+ "glide_duration_ms":{"type":"number","minimum":50,"maximum":5000,"description":"Flight duration per move in ms. Default 160."},+ "dwell_after_click_ms":{"type":"number","minimum":0,"maximum":5000,"description":"Pause after click ripple in ms. Default 80."},+ "idle_hide_ms":{"type":"number","minimum":0,"maximum":60000,"description":"Auto-hide delay in ms. 0=never. Default 20000."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ // JSON numbers without decimals parse as ints; coerce to f64 so+ // callers can write `{"glide_duration_ms": 1500}` without it being+ // silently ignored (matches Swift's `number()` helper).+ fn num(v: Option<&Value>) -> Option<f64> {+ v.and_then(|x| x.as_f64().or_else(|| x.as_i64().map(|i| i as f64)))+ }+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned();+ // 1. Per-instance appearance fields (Rust-only).+ self.state.cursor_registry.update_config(&cursor_id, |cfg| {+ if let Some(v) = args.get("cursor_icon").and_then(|v| v.as_str()) { cfg.cursor_icon = Some(v.to_owned()); }+ if let Some(v) = args.get("cursor_color").and_then(|v| v.as_str()) { cfg.cursor_color = Some(v.to_owned()); }+ if let Some(v) = args.get("cursor_label").and_then(|v| v.as_str()) { cfg.cursor_label = Some(v.to_owned()); }+ if let Some(v) = num(args.get("cursor_size")) { cfg.cursor_size = Some(v); }+ if let Some(v) = num(args.get("cursor_opacity")) { cfg.cursor_opacity = Some(v.clamp(0.0, 1.0)); }+ });+ // 2. Apply motion knobs to the live render state — was silently+ // dropped before; this is the Swift parity behavior.+ let current = crate::overlay::current_motion();+ let updated = current.with_overrides(+ num(args.get("start_handle")),+ num(args.get("end_handle")),+ num(args.get("arc_size")),+ num(args.get("arc_flow")),+ num(args.get("spring")),+ num(args.get("glide_duration_ms")),+ num(args.get("dwell_after_click_ms")),+ num(args.get("idle_hide_ms")),+ None, // press_duration_ms — not in Swift tool surface+ );+ crate::overlay::send_command(cursor_overlay::OverlayCommand::SetMotion(updated.clone()));+ // Match Swift text format 1:1.+ let summary = format!(+ "cursor motion: startHandle={sh} endHandle={eh} arcSize={asz} arcFlow={af} \+ spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih}",+ sh = updated.start_handle, eh = updated.end_handle,+ asz = updated.arc_size, af = updated.arc_flow,+ sp = updated.spring,+ gd = updated.glide_duration_ms as i64,+ dw = updated.dwell_after_click_ms as i64,+ ih = updated.idle_hide_ms as i64,+ );+ ToolResult::text(format!("✅ {summary}")).with_structured(json!({+ "cursor_id": cursor_id,+ "start_handle": updated.start_handle,+ "end_handle": updated.end_handle,+ "arc_size": updated.arc_size,+ "arc_flow": updated.arc_flow,+ "spring": updated.spring,+ "glide_duration_ms": updated.glide_duration_ms,+ "dwell_after_click_ms": updated.dwell_after_click_ms,+ "idle_hide_ms": updated.idle_hide_ms,+ }))+ }+}++// ── get_agent_cursor_state ────────────────────────────────────────────────────++pub struct GetAgentCursorStateTool {+ state: Arc<ToolState>,+}++static GCSTATE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetAgentCursorStateTool {+ fn def(&self) -> &ToolDef {+ GCSTATE_DEF.get_or_init(|| ToolDef {+ name: "get_agent_cursor_state".into(),+ // Description ported from Swift `GetAgentCursorStateTool.swift`.+ description: "Report the current agent-cursor configuration: enabled flag, \+ motion knobs (startHandle, endHandle, arcSize, arcFlow, spring), glide \+ duration, post-click dwell, and idle-hide delay. Durations come back in \+ milliseconds to match the setter's units. Pure read-only — no side effects.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ let enabled = crate::overlay::is_enabled();+ let motion = crate::overlay::current_motion();+ // Swift text format 1:1: single-line camelCase key=value pairs.+ let summary = format!(+ "cursor: enabled={enabled} startHandle={sh} endHandle={eh} arcSize={asz} \+ arcFlow={af} spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih}",+ sh = motion.start_handle, eh = motion.end_handle,+ asz = motion.arc_size, af = motion.arc_flow,+ sp = motion.spring,+ gd = motion.glide_duration_ms as i64,+ dw = motion.dwell_after_click_ms as i64,+ ih = motion.idle_hide_ms as i64,+ );+ // Rust-only structured payload: the same fields + the multi-cursor+ // instance map. Cursor instances are a Rust extension Swift doesn't+ // expose; documented in PARITY.md.+ let cursors = serde_json::to_value(self.state.cursor_registry.all_states()).unwrap_or_default();+ ToolResult::text(format!("✅ {summary}"))+ .with_structured(json!({+ "enabled": enabled,+ "start_handle": motion.start_handle,+ "end_handle": motion.end_handle,+ "arc_size": motion.arc_size,+ "arc_flow": motion.arc_flow,+ "spring": motion.spring,+ "glide_duration_ms": motion.glide_duration_ms,+ "dwell_after_click_ms": motion.dwell_after_click_ms,+ "idle_hide_ms": motion.idle_hide_ms,+ "cursors": cursors,+ }))+ }+}++// ── set_agent_cursor_style ────────────────────────────────────────────────────++pub struct SetAgentCursorStyleTool {+ state: Arc<ToolState>,+}++static STYLE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetAgentCursorStyleTool {+ fn def(&self) -> &ToolDef {+ STYLE_DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_style".into(),+ description:+ "Update the visual style of the agent cursor overlay.\n\n\+ - gradient_colors: array of CSS hex strings (e.g. [\"#FF0000\",\"#0000FF\"]) \+ used as the arrow fill gradient from tip to tail. Empty array reverts to \+ the default palette colours.\n\+ - bloom_color: hex string for the radial halo/bloom behind the cursor \+ (e.g. \"#00FFFF\"). Empty string reverts to the default.\n\+ - image_path: path to a PNG, JPEG, SVG, or ICO file to use as the cursor \+ icon instead of the default gradient arrow. Empty string reverts to the \+ procedural arrow.\n\+ All parameters are optional; omit any you do not want to change."+ .into(),+ input_schema: json!({+ "type": "object",+ "properties": {+ "cursor_id": {+ "type": "string",+ "description": "Cursor instance. Default: 'default'."+ },+ "gradient_colors": {+ "type": "array",+ "items": { "type": "string" },+ "description": "CSS hex gradient stops tip→tail. [] = revert to default."+ },+ "bloom_color": {+ "type": "string",+ "description": "Hex bloom/halo colour (e.g. '#00FFFF'). '' = revert to default."+ },+ "image_path": {+ "type": "string",+ "description": "Path to PNG/JPEG/SVG/ICO cursor image. '' = revert to arrow."+ }+ },+ "additionalProperties": false+ }),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned();++ // image_path+ let image_path = args.get("image_path").and_then(|v| v.as_str());+ let shape_cmd: Option<cursor_overlay::OverlayCommand> = if let Some(path) = image_path {+ if path.is_empty() {+ Some(cursor_overlay::OverlayCommand::SetShape(None))+ } else {+ let path_owned = path.to_owned();+ match tokio::task::spawn_blocking(move || {+ cursor_overlay::CursorShape::load(&path_owned)+ }).await {+ Ok(Ok(shape)) => {+ let path_owned2 = path.to_owned();+ self.state.cursor_registry.update_config(&cursor_id, |c| {+ c.cursor_icon = Some(path_owned2);+ });+ Some(cursor_overlay::OverlayCommand::SetShape(Some(shape)))+ }+ Ok(Err(e)) => return ToolResult::error(format!("Failed to load image_path: {e}")),+ Err(e) => return ToolResult::error(format!("Task error: {e}")),+ }+ }+ } else {+ None+ };++ // gradient_colors+ let gradient_colors: Vec<[u8; 4]> = if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {+ let mut out = vec![];+ for v in arr {+ if let Some(hex) = v.as_str() {+ match parse_hex_color(hex) {+ Some(c) => out.push(c),+ None => return ToolResult::error(format!("Invalid hex color: {hex}")),+ }+ }+ }+ out+ } else {+ vec![]+ };++ // bloom_color+ let bloom_color: Option<Option<[u8; 4]>> = if let Some(hex) = args.get("bloom_color").and_then(|v| v.as_str()) {+ if hex.is_empty() {+ Some(None)+ } else {+ match parse_hex_color(hex) {+ Some(c) => Some(Some(c)),+ None => return ToolResult::error(format!("Invalid bloom_color: {hex}")),+ }+ }+ } else {+ None+ };++ // Dispatch to overlay+ if let Some(cmd) = shape_cmd {+ crate::overlay::send_command(cmd);+ }+ let gradient_provided = args.get("gradient_colors").is_some();+ let bloom_provided = args.get("bloom_color").is_some();+ if gradient_provided || bloom_provided {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::SetGradient {+ gradient_colors,+ bloom_color: bloom_color.flatten(),+ });+ }++ // Swift `SetAgentCursorStyleTool` text format: only include fields+ // whose post-write value is `Some` (i.e. not reverted to default).+ // Falls back to "✅ cursor style: reverted to default" when every+ // field is empty.+ let mut parts: Vec<String> = Vec::new();+ if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {+ let hexes: Vec<String> = arr.iter()+ .filter_map(|v| v.as_str().map(str::to_owned))+ .collect();+ if !hexes.is_empty() {+ parts.push(format!("gradient_colors=[{}]", hexes.join(",")));+ }+ }+ if let Some(s) = args.get("bloom_color").and_then(|v| v.as_str()) {+ if !s.is_empty() { parts.push(format!("bloom_color={s}")); }+ }+ if let Some(s) = image_path {+ if !s.is_empty() { parts.push(format!("image_path={s}")); }+ }+ let summary = if parts.is_empty() {+ "reverted to default".to_owned()+ } else {+ parts.join(" ")+ };+ ToolResult::text(format!("✅ cursor style: {summary}"))+ }+}++fn parse_hex_color(hex: &str) -> Option<[u8; 4]> {+ let s = hex.trim_start_matches('#');+ match s.len() {+ 6 => {+ let r = u8::from_str_radix(&s[0..2], 16).ok()?;+ let g = u8::from_str_radix(&s[2..4], 16).ok()?;+ let b = u8::from_str_radix(&s[4..6], 16).ok()?;+ Some([r, g, b, 255])+ }+ 3 => {+ let r = u8::from_str_radix(&s[0..1].repeat(2), 16).ok()?;+ let g = u8::from_str_radix(&s[1..2].repeat(2), 16).ok()?;+ let b = u8::from_str_radix(&s[2..3].repeat(2), 16).ok()?;+ Some([r, g, b, 255])+ }+ _ => None,+ }+}++// ── check_permissions ─────────────────────────────────────────────────────────++pub struct CheckPermissionsTool;+static PERMS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for CheckPermissionsTool {+ fn def(&self) -> &ToolDef {+ PERMS_DEF.get_or_init(|| ToolDef {+ name: "check_permissions".into(),+ description: "Check required permissions for cua-driver-rs on Windows.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ // On Windows, background PostMessage access doesn't require elevated permissions.+ // UIA works for most apps without elevation; some system apps require elevation.+ use windows::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken};+ use windows::Win32::Security::{TOKEN_QUERY, GetTokenInformation, TokenElevation, TOKEN_ELEVATION};+ let is_elevated = unsafe {+ let proc = GetCurrentProcess();+ let mut token = windows::Win32::Foundation::HANDLE::default();+ if OpenProcessToken(proc, TOKEN_QUERY, &mut token).is_ok() {+ let mut elevation = TOKEN_ELEVATION::default();+ let mut ret_len = 0u32;+ let ok = GetTokenInformation(+ token,+ TokenElevation,+ Some(&mut elevation as *mut _ as *mut _),+ std::mem::size_of::<TOKEN_ELEVATION>() as u32,+ &mut ret_len,+ ).is_ok();+ let _ = windows::Win32::Foundation::CloseHandle(token);+ ok && elevation.TokenIsElevated != 0+ } else {+ false+ }+ };+ let status_text = format!(+ "Process elevation: {}\nUIA accessibility: available (no special permission needed)\nPostMessage injection: available",+ if is_elevated { "✅ elevated (administrator)" } else { "ℹ️ standard user" }+ );+ ToolResult::text(status_text)+ .with_structured(json!({ "elevated": is_elevated, "uia": true, "post_message": true }))+ }+}++// ── get_config ────────────────────────────────────────────────────────────────++pub struct GetConfigTool {+ state: Arc<ToolState>,+}+static GCFG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetConfigTool {+ fn def(&self) -> &ToolDef {+ GCFG_DEF.get_or_init(|| ToolDef {+ name: "get_config".into(),+ // Description ported from Swift `GetConfigTool.swift` with the+ // Windows config path. Swift's `agent_cursor.*` subtree is not+ // mirrored on Windows yet (separate cursor-overlay config path);+ // documented in PARITY.md.+ description: "Report the current persistent driver config.\n\n\+ Pure read-only. Returns defaults when the underlying state is unset — \+ same fallback the daemon uses at startup. Sibling to `set_config`.\n\n\+ Current schema (Windows):\n\n \+ {\n \"schema_version\": 1,\n \"version\": \"<crate version>\",\n \+ \"platform\": \"windows\",\n \"capture_mode\": \"som\" | \"vision\" | \"ax\",\n \+ \"max_image_dimension\": 0\n }\n".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ let cfg = self.state.config.read().unwrap();+ // Mirror the macOS agent's parity addition (commit adb9ecca):+ // nested `agent_cursor.enabled` block so Swift-shaped get_config+ // consumers can read the cursor's enabled state from one place.+ let cursor_enabled = self.state.cursor_registry.all_states()+ .first()+ .map(|s| s.config.enabled)+ .unwrap_or(true);+ let payload = json!({+ "schema_version": 1,+ "version": env!("CARGO_PKG_VERSION"),+ "platform": "windows",+ "capture_mode": cfg.capture_mode,+ "max_image_dimension": cfg.max_image_dimension,+ "agent_cursor": { "enabled": cursor_enabled },+ });+ // Match Swift's text format 1:1: `"✅ <pretty JSON>"`.+ let pretty = serde_json::to_string_pretty(&payload).unwrap_or_else(|_| payload.to_string());+ ToolResult::text(format!("✅ {pretty}")).with_structured(payload)+ }+}++// ── set_config ────────────────────────────────────────────────────────────────++pub struct SetConfigTool {+ state: Arc<ToolState>,+}+static SCFG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for SetConfigTool {+ fn def(&self) -> &ToolDef {+ SCFG_DEF.get_or_init(|| ToolDef {+ name: "set_config".into(),+ // Description ported from Swift `SetConfigTool.swift`. Windows+ // accepts BOTH Swift's `{key, value}` dotted-path shape AND a+ // legacy per-field shape; documented as intentional.+ description: "Write a setting into the persistent driver config. Values take \+ effect immediately.\n\n\+ Two input shapes:\n\+ - **Swift-compatible** (preferred): `{\"key\": \"capture_mode\", \"value\": \"som\"}` \+ — single dotted-path leaf write.\n\+ - **Legacy per-field** (Rust-only): `{\"capture_mode\": \"som\", \"max_image_dimension\": 0}` \+ — bulk write of named fields.\n\n\+ Known keys:\n\+ - `capture_mode` (string: `vision` | `ax` | `som`)\n\+ - `max_image_dimension` (integer)\n\n\+ Returns the full updated config in the same shape as `get_config`.".into(),+ input_schema: json!({"type":"object","properties":{+ "key":{"type":"string","description":"Dotted snake_case path to a leaf config field (Swift-compatible shape). Pair with `value`."},+ "value":{"description":"New value for `key`. JSON type depends on the key."},+ "capture_mode":{"type":"string","enum":["som","vision","ax"],"description":"Legacy per-field shape."},+ "max_image_dimension":{"type":"integer","description":"Legacy per-field shape."}+ },"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, args: Value) -> ToolResult {+ let mut cfg = self.state.config.write().unwrap();+ let mut applied = false;+ // Swift-compatible {key, value} shape.+ if let (Some(key), Some(val)) = (+ args.get("key").and_then(|v| v.as_str()),+ args.get("value"),+ ) {+ match key {+ "capture_mode" => match val.as_str() {+ Some(s) => { cfg.capture_mode = s.to_owned(); applied = true; }+ None => return ToolResult::error(format!("`capture_mode` must be a string, got {val}.")),+ },+ "max_image_dimension" => match val.as_u64() {+ Some(n) => { cfg.max_image_dimension = n as u32; applied = true; }+ None => return ToolResult::error(format!("`max_image_dimension` must be an integer, got {val}.")),+ },+ other => return ToolResult::error(format!(+ "Unknown config key `{other}`. Known: capture_mode, max_image_dimension."+ )),+ }+ }+ // Legacy per-field shape.+ if let Some(mode) = args.get("capture_mode").and_then(|v| v.as_str()) {+ cfg.capture_mode = mode.to_owned(); applied = true;+ }+ if let Some(dim) = args.get("max_image_dimension").and_then(|v| v.as_u64()) {+ cfg.max_image_dimension = dim as u32; applied = true;+ }+ if !applied {+ return ToolResult::error("Missing required string field `key` (or a known legacy per-field).");+ }+ // Emit the same pretty-JSON payload as `get_config` (matches Swift's+ // `set_config` return shape — both tools echo the full config after).+ let cursor_enabled = self.state.cursor_registry.all_states()+ .first()+ .map(|s| s.config.enabled)+ .unwrap_or(true);+ let payload = json!({+ "schema_version": 1,+ "version": env!("CARGO_PKG_VERSION"),+ "platform": "windows",+ "capture_mode": cfg.capture_mode,+ "max_image_dimension": cfg.max_image_dimension,+ "agent_cursor": { "enabled": cursor_enabled },+ });+ let pretty = serde_json::to_string_pretty(&payload).unwrap_or_else(|_| payload.to_string());+ ToolResult::text(format!("✅ {pretty}")).with_structured(payload)+ }+}++// ── get_accessibility_tree ────────────────────────────────────────────────────++pub struct GetAccessibilityTreeTool;++static GAX_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetAccessibilityTreeTool {+ fn def(&self) -> &ToolDef {+ GAX_DEF.get_or_init(|| ToolDef {+ name: "get_accessibility_tree".into(),+ description: "Return a lightweight snapshot of the desktop: running processes and \+ on-screen visible windows with their bounds and owner pid.\n\n\+ For the full UIA subtree of a single window (with interactive element indices \+ you can click by), use get_window_state instead — this is a fast discovery read.".into(),+ input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: true, open_world: false,+ })+ }+ async fn invoke(&self, _args: Value) -> ToolResult {+ let (procs, windows) = tokio::task::spawn_blocking(|| {+ (crate::win32::list_processes(), crate::win32::list_windows(None))+ }).await.unwrap_or_default();++ let mut lines = vec![format!(+ "{} running process(es), {} visible window(s)",+ procs.len(), windows.len()+ )];+ for p in &procs {+ lines.push(format!("- {} (pid {})", p.name, p.pid));+ }+ if !windows.is_empty() {+ lines.push(String::new());+ lines.push("Windows:".to_owned());+ for w in &windows {+ let title = if w.title.is_empty() { "(no title)".to_owned() }+ else { format!("\"{}\"", w.title) };+ lines.push(format!(+ "- pid={} {} [window_id: {}] {}x{}+{}+{}",+ w.pid, title, w.hwnd, w.width, w.height, w.x, w.y+ ));+ }+ lines.push("→ Call get_window_state(pid, window_id) to inspect a window's UI.".to_owned());+ }++ let structured = json!({+ "processes": procs.iter().map(|p| json!({"pid":p.pid,"name":p.name})).collect::<Vec<_>>(),+ "windows": windows.iter().map(|w| json!({+ "window_id": w.hwnd, "pid": w.pid, "title": w.title,+ "x": w.x, "y": w.y, "width": w.width, "height": w.height+ })).collect::<Vec<_>>()+ });+ ToolResult::text(lines.join("\n")).with_structured(structured)+ }+}++// ── zoom ──────────────────────────────────────────────────────────────────────++pub struct ZoomTool {+ state: Arc<ToolState>,+}+static ZOOM_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for ZoomTool {+ fn def(&self) -> &ToolDef {+ ZOOM_DEF.get_or_init(|| ToolDef {+ name: "zoom".into(),+ // Description ported from Swift `ZoomTool.swift`.+ description: "Zoom into a rectangular region of a window screenshot at full (native) \+ resolution. Use this when `get_window_state` returned a resized image and you \+ need to read small text, identify icons, or verify UI details.\n\n\+ Coordinates `x1, y1, x2, y2` are in the same pixel space as the screenshot \+ returned by `get_window_state` (i.e. the resized image if `max_image_dimension` \+ is active). The maximum zoom region width is 500 px in scaled-image coordinates.\n\n\+ A 20% padding is automatically added on every side of the requested region so \+ the target remains visible even if the caller's coordinates are slightly off.\n\n\+ After a zoom, pass `from_zoom=true` to click/type_text to auto-translate \+ coordinates back to full-window space.\n\n\+ Windows-specific: `window_id` is the canonical addressing field (HWND). \+ `pid` is also required (used to scope the zoom translation registry, matching \+ Swift's per-pid zoom context).".into(),+ input_schema: json!({+ "type":"object","required":["pid","window_id","x1","y1","x2","y2"],"properties":{+ "pid":{"type":"integer","description":"Target process ID."},+ "window_id":{"type":"integer","description":"HWND of the target window."},+ "x1":{"type":"number","description":"Left edge of the region (resized-image pixels)."},+ "y1":{"type":"number","description":"Top edge of the region (resized-image pixels)."},+ "x2":{"type":"number","description":"Right edge of the region (resized-image pixels)."},+ "y2":{"type":"number","description":"Bottom edge of the region (resized-image pixels)."}+ },"additionalProperties":false+ }),+ // Swift annotation: idempotent: false.+ read_only: true, destructive: false, idempotent: false, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {+ Some(p) => p,+ None => return ToolResult::error("Missing required integer field pid."),+ };+ let pid = Some(raw_pid as u32);+ let hwnd = match args.get("window_id").and_then(|v| v.as_u64()) {+ Some(v) => v,+ None => return ToolResult::error("Missing required integer field window_id."),+ };+ let coerce = |k: &str| args.get(k).and_then(|v| v.as_f64().or_else(|| v.as_i64().map(|i| i as f64)));+ let (x1, y1, x2, y2) = match (coerce("x1"), coerce("y1"), coerce("x2"), coerce("y2")) {+ (Some(a), Some(b), Some(c), Some(d)) => (a, b, c, d),+ _ => return ToolResult::error("Missing required region coordinates (x1, y1, x2, y2)."),+ };+ if x2 <= x1 || y2 <= y1 {+ return ToolResult::error("Invalid region: x2 must be > x1 and y2 must be > y1.");+ }+ if x2 - x1 > 500.0 {+ return ToolResult::error(format!(+ "Zoom region too wide: {} px > 500 px max. Use a narrower region (max 500 px in scaled-image coordinates).",+ (x2 - x1) as i64+ ));+ }++ let state = self.state.clone();+ let result = tokio::task::spawn_blocking(move || {+ let png = crate::capture::screenshot_window_bytes(hwnd)?;+ cursor_overlay::capture_utils::crop_png_to_jpeg(&png, x1, y1, x2, y2, 500)+ }).await;++ match result {+ Ok(Ok(crop)) => {+ if let Some(p) = pid {+ state.zoom_registry.set(p, ZoomContext {+ origin_x: crop.origin_x,+ origin_y: crop.origin_y,+ scale_inv: crop.scale_inv,+ });+ }+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};+ let b64 = B64.encode(&crop.jpeg_bytes);+ let (w, h) = (crop.out_w, crop.out_h);+ use mcp_server::protocol::Content;+ // Match Swift's text format 1:1.+ let summary = "✅ Zoomed region captured at native resolution. \+ To click a target in this image, use \+ `click(pid, x, y, from_zoom=true)` where x,y are pixel \+ coordinates in THIS zoomed image — the driver maps them \+ back automatically.".to_owned();+ ToolResult {+ content: vec![+ Content::image_jpeg(b64),+ Content::text(summary),+ ],+ is_error: None,+ structured_content: Some(json!({ "width": w, "height": h, "format": "jpeg" })),+ }+ }+ Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── type_text_chars ───────────────────────────────────────────────────────────++pub struct TypeTextCharsTool { state: Arc<ToolState> }+static TYPE_CHARS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for TypeTextCharsTool {+ fn def(&self) -> &ToolDef {+ TYPE_CHARS_DEF.get_or_init(|| ToolDef {+ name: "type_text_chars".into(),+ description: "Type text character-by-character with a configurable inter-character \+ delay (default 30 ms). Useful for apps that miss keystrokes. \+ Otherwise identical to type_text (WM_CHAR, no focus steal).".into(),+ input_schema: json!({+ "type":"object","required":["pid","text"],"properties":{+ "pid":{"type":"integer"},+ "window_id":{"type":"integer"},+ "text":{"type":"string"},+ "delay_ms":{"type":"integer","description":"Milliseconds between chars (default 30)."},+ "element_index":{"type":"integer"},+ "type_chars_only":{"type":"boolean","description":"Skip element focus, type directly. Default false."}+ },"additionalProperties":false+ }),+ read_only: false, destructive: true, idempotent: false, open_world: true,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32;+ let text = match args.get("text").and_then(|v| v.as_str()) {+ Some(t) => t.to_owned(), None => return ToolResult::error("Missing required parameter: text"),+ };+ let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30);+ let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());+ let hwnd = match hwnd_opt {+ Some(h) => h,+ None => {+ let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();+ match windows.first() {+ Some(w) => w.hwnd,+ None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),+ }+ }+ };+ let text_len = text.chars().count();+ let result = tokio::task::spawn_blocking(move || {+ crate::input::post_type_text_with_delay(hwnd, &text, delay_ms)+ }).await;+ match result {+ Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) with {delay_ms}ms delay.")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}++// ── registry builder ──────────────────────────────────────────────────────────++pub fn build_registry() -> ToolRegistry {+ let state = ToolState::new();+ let mut r = ToolRegistry::new();+ r.register(Box::new(ListAppsTool));+ r.register(Box::new(ListWindowsTool));+ r.register(Box::new(GetWindowStateTool { state: state.clone() }));+ r.register(Box::new(LaunchAppTool));+ r.register(Box::new(ClickTool { state: state.clone() }));+ r.register(Box::new(DoubleClickTool { state: state.clone() }));+ r.register(Box::new(RightClickTool { state: state.clone() }));+ r.register(Box::new(DragTool { state: state.clone() }));+ r.register(Box::new(TypeTextTool { state: state.clone() }));+ r.register(Box::new(PressKeyTool));+ r.register(Box::new(HotkeyTool));+ r.register(Box::new(SetValueTool { state: state.clone() }));+ r.register(Box::new(ScrollTool));+ r.register(Box::new(ScreenshotTool { state: state.clone() }));+ r.register(Box::new(GetScreenSizeTool));+ r.register(Box::new(GetCursorPositionTool));+ r.register(Box::new(MoveCursorTool { state: state.clone() }));+ r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() }));+ r.register(Box::new(SetAgentCursorMotionTool { state: state.clone() }));+ r.register(Box::new(GetAgentCursorStateTool { state: state.clone() }));+ r.register(Box::new(SetAgentCursorStyleTool { state: state.clone() }));+ r.register(Box::new(CheckPermissionsTool));+ r.register(Box::new(GetConfigTool { state: state.clone() }));+ r.register(Box::new(SetConfigTool { state: state.clone() }));+ r.register(Box::new(GetAccessibilityTreeTool));+ r.register(Box::new(ZoomTool { state: state.clone() }));+ // `type_text_chars` is intentionally NOT registered — Swift treats it+ // as a deprecated alias for `type_text` resolved at invoke time in+ // mcp-server's `ToolRegistry::invoke`. Keeping it out of the registry+ // means it doesn't show up in `tools/list` either, matching Swift's+ // ToolRegistry.swift (`type_text_chars` not in `handlers`).+ let _: &TypeTextCharsTool = &TypeTextCharsTool { state: state.clone() }; // touch struct so it stays in this crate for now+ r.register_recording_tools();+ r+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/mod.rsnew file mode 100644index 0000000000..c808d1efed--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/mod.rs@@ -0,0 +1,20 @@+//! Windows tool implementations.+//!+//! On Windows, delegates to real win32/uia/input/capture implementations.+//! On other platforms, returns "not implemented" stubs so the crate still compiles.++use mcp_server::tool::ToolRegistry;++#[cfg(target_os = "windows")]+mod impl_;++#[cfg(not(target_os = "windows"))]+mod stubs;++pub fn build_registry() -> ToolRegistry {+ #[cfg(target_os = "windows")]+ return impl_::build_registry();++ #[cfg(not(target_os = "windows"))]+ stubs::build_registry()+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/stubs.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/stubs.rsnew file mode 100644index 0000000000..ce274f2bd9--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/stubs.rs@@ -0,0 +1,227 @@+//! Non-Windows stubs — return a helpful "not implemented" message.+//! Schemas match the macOS platform-macos implementations (cross-platform interface).++use async_trait::async_trait;+use mcp_server::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};+use serde_json::Value;++fn not_impl(name: &str) -> ToolResult {+ ToolResult::error(format!(+ "{name}: Windows implementation is only available when compiled for Windows (target_os = \"windows\")."+ ))+}++macro_rules! stub_tool {+ ($mod_name:ident, $struct_name:ident, $tool_name:literal, $desc:literal, $schema:expr) => {+ mod $mod_name {+ use super::*;+ pub struct $struct_name;+ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+ impl $struct_name {+ pub fn def_static() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: $tool_name.into(),+ description: $desc.into(),+ input_schema: $schema,+ read_only: false,+ destructive: false,+ idempotent: false,+ open_world: false,+ })+ }+ }+ #[async_trait]+ impl Tool for $struct_name {+ fn def(&self) -> &ToolDef { Self::def_static() }+ async fn invoke(&self, _args: Value) -> ToolResult { not_impl($tool_name) }+ }+ }+ pub use $mod_name::$struct_name;+ };+}++stub_tool!(list_apps_m, ListAppsTool, "list_apps",+ "List running regular-UI applications.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(list_windows_m, ListWindowsTool, "list_windows",+ "List top-level windows, optionally filtered by pid.",+ serde_json::json!({"type":"object","properties":{"pid":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(get_window_state_m, GetWindowStateTool, "get_window_state",+ "Walk a running app's AX/UIA tree and return a Markdown rendering of its UI. Also captures a screenshot.",+ serde_json::json!({"type":"object","required":["pid","window_id"],"properties":{"pid":{"type":"integer"},"window_id":{"type":"integer"},"query":{"type":"string"},"capture_mode":{"type":"string","enum":["som","vision","ax"]}},"additionalProperties":false}));++stub_tool!(launch_app_m, LaunchAppTool, "launch_app",+ "Launch an app in the background. Provide bundle_id (macOS) or name.",+ serde_json::json!({"type":"object","properties":{"bundle_id":{"type":"string"},"name":{"type":"string"},"urls":{"type":"array","items":{"type":"string"}}},"additionalProperties":false}));++stub_tool!(click_m, ClickTool, "click",+ "Click at (x, y) or on an AX/UIA element by element_index + window_id.",+ serde_json::json!({"type":"object","required":["pid"],"properties":{"pid":{"type":"integer"},"window_id":{"type":"integer"},"element_index":{"type":"integer"},"x":{"type":"number"},"y":{"type":"number"},"modifier":{"type":"array","items":{"type":"string"}},"from_zoom":{"type":"boolean"}},"additionalProperties":false}));++stub_tool!(double_click_m, DoubleClickTool, "double_click",+ "Double-click at (x, y) or on an AX/UIA element by element_index + window_id.",+ serde_json::json!({"type":"object","required":["pid"],"properties":{"pid":{"type":"integer"},"x":{"type":"number"},"y":{"type":"number"},"window_id":{"type":"integer"},"element_index":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(right_click_m, RightClickTool, "right_click",+ "Right-click (AXShowMenu on element, or synthesized event at x,y).",+ serde_json::json!({"type":"object","required":["pid"],"properties":{"pid":{"type":"integer"},"element_index":{"type":"integer"},"window_id":{"type":"integer"},"x":{"type":"number"},"y":{"type":"number"},"modifier":{"type":"array","items":{"type":"string"}}},"additionalProperties":false}));++stub_tool!(type_text_m, TypeTextTool, "type_text",+ "Insert text into the target pid via AXSelectedText or WM_CHAR. Falls back to CGEvent/keystrokes.",+ serde_json::json!({"type":"object","required":["pid","text"],"properties":{"pid":{"type":"integer"},"text":{"type":"string"},"element_index":{"type":"integer"},"window_id":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(type_chars_m, TypeTextCharsTool, "type_text_chars",+ "Type text character-by-character with configurable per-character delay.",+ serde_json::json!({"type":"object","required":["pid","text"],"properties":{"pid":{"type":"integer"},"text":{"type":"string"},"delay_ms":{"type":"integer"},"window_id":{"type":"integer"},"element_index":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(press_key_m, PressKeyTool, "press_key",+ "Press and release a single key delivered directly to the target pid. No focus steal.",+ serde_json::json!({"type":"object","required":["pid","key"],"properties":{"pid":{"type":"integer"},"key":{"type":"string"},"modifiers":{"type":"array","items":{"type":"string"}},"window_id":{"type":"integer"},"element_index":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(hotkey_m, HotkeyTool, "hotkey",+ "Press a combination of keys simultaneously, e.g. [\"cmd\",\"c\"] for Copy.",+ serde_json::json!({"type":"object","required":["pid","keys"],"properties":{"pid":{"type":"integer"},"keys":{"type":"array","items":{"type":"string"},"minItems":2}},"additionalProperties":false}));++stub_tool!(set_value_m, SetValueTool, "set_value",+ "Set the value of an AX/UIA element (text field, dropdown, checkbox).",+ serde_json::json!({"type":"object","required":["pid","window_id","element_index","value"],"properties":{"pid":{"type":"integer"},"window_id":{"type":"integer"},"element_index":{"type":"integer"},"value":{"type":"string"}},"additionalProperties":false}));++stub_tool!(scroll_m, ScrollTool, "scroll",+ "Scroll the target pid's focused region. direction required; by defaults to line, amount defaults to 3.",+ serde_json::json!({"type":"object","required":["pid","direction"],"properties":{"pid":{"type":"integer"},"direction":{"type":"string","enum":["up","down","left","right"]},"by":{"type":"string","enum":["line","page"]},"amount":{"type":"integer","minimum":1,"maximum":50},"window_id":{"type":"integer"},"element_index":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(screenshot_m, ScreenshotTool, "screenshot",+ "Capture a screenshot. Without window_id captures the full display. Supports png and jpeg formats.",+ serde_json::json!({"type":"object","properties":{"window_id":{"type":"integer"},"format":{"type":"string","enum":["png","jpeg"]},"quality":{"type":"integer","minimum":1,"maximum":95}},"additionalProperties":false}));++stub_tool!(get_screen_size_m, GetScreenSizeTool, "get_screen_size",+ "Return the main screen's width and height in points.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(get_cursor_position_m, GetCursorPositionTool, "get_cursor_position",+ "Return the current real mouse cursor position.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(cursor_motion_m, SetAgentCursorMotionTool, "set_agent_cursor_motion",+ "Configure the visual appearance of an agent cursor instance.\n\nExtended cursor customization for multi-cursor use cases:\n- cursor_id: instance name (default='default')\n- cursor_icon: built-in ('arrow','crosshair','hand','dot') or PNG/SVG file path\n- cursor_color: hex color e.g. '#00FFFF' or CSS name\n- cursor_label: short text shown near the cursor\n- cursor_size: dot radius in points (default=16)\n- cursor_opacity: 0.0–1.0 (default=0.85)",+ serde_json::json!({"type":"object","properties":{"cursor_id":{"type":"string"},"cursor_icon":{"type":"string"},"cursor_color":{"type":"string"},"cursor_label":{"type":"string"},"cursor_size":{"type":"number"},"cursor_opacity":{"type":"number"}},"additionalProperties":false}));++stub_tool!(get_cursor_state_m, GetAgentCursorStateTool, "get_agent_cursor_state",+ "Return the current state of all agent cursor instances.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(set_cursor_style_m, SetAgentCursorStyleTool, "set_agent_cursor_style",+ "Update the visual style of the agent cursor overlay.",+ serde_json::json!({"type":"object","properties":{"cursor_id":{"type":"string"},"gradient_colors":{"type":"array","items":{"type":"string"}},"bloom_color":{"type":"string"},"image_path":{"type":"string"}},"additionalProperties":false}));++stub_tool!(check_perms_m, CheckPermissionsTool, "check_permissions",+ "Check required permissions (Accessibility, Screen Recording).",+ serde_json::json!({"type":"object","properties":{"prompt":{"type":"boolean"}},"additionalProperties":false}));++stub_tool!(get_config_m, GetConfigTool, "get_config",+ "Return current cua-driver-rs configuration.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(set_config_m, SetConfigTool, "set_config",+ "Update cua-driver-rs configuration.",+ serde_json::json!({"type":"object","properties":{"capture_mode":{"type":"string","enum":["som","vision","ax"]},"max_image_dimension":{"type":"integer"}},"additionalProperties":false}));++stub_tool!(get_ax_tree_m, GetAccessibilityTreeTool, "get_accessibility_tree",+ "Lightweight desktop snapshot: running apps and visible windows. For per-window AX tree, use get_window_state.",+ serde_json::json!({"type":"object","properties":{},"additionalProperties":false}));++stub_tool!(zoom_m, ZoomTool, "zoom",+ "Crop a window region to JPEG with 20% padding, max 500px wide.",+ serde_json::json!({"type":"object","required":["window_id","x1","y1","x2","y2"],"properties":{"window_id":{"type":"integer"},"x1":{"type":"number"},"y1":{"type":"number"},"x2":{"type":"number"},"y2":{"type":"number"},"pid":{"type":"integer"}},"additionalProperties":false}));++// move_cursor and set_agent_cursor_enabled route through the overlay channel.++mod move_cursor_m {+ use super::*;+ pub struct MoveCursorTool;+ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+ impl MoveCursorTool {+ pub fn def_static() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "move_cursor".into(),+ description: "Move the agent cursor overlay to (x, y). Does NOT move the real \+ mouse cursor — the user's cursor stays where it is.".into(),+ input_schema: serde_json::json!({"type":"object","required":["x","y"],"properties":{"x":{"type":"number"},"y":{"type":"number"},"cursor_id":{"type":"string"}},"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ }+ #[async_trait]+ impl Tool for MoveCursorTool {+ fn def(&self) -> &ToolDef { Self::def_static() }+ async fn invoke(&self, args: Value) -> ToolResult {+ let x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo { x, y, end_heading_radians: 0.0 });+ ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1})."))+ }+ }+}+pub use move_cursor_m::MoveCursorTool;++mod set_enabled_m {+ use super::*;+ pub struct SetAgentCursorEnabledTool;+ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+ impl SetAgentCursorEnabledTool {+ pub fn def_static() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "set_agent_cursor_enabled".into(),+ description: "Show or hide the agent cursor overlay.".into(),+ input_schema: serde_json::json!({"type":"object","required":["enabled"],"properties":{"enabled":{"type":"boolean"},"cursor_id":{"type":"string"}},"additionalProperties":false}),+ read_only: false, destructive: false, idempotent: true, open_world: false,+ })+ }+ }+ #[async_trait]+ impl Tool for SetAgentCursorEnabledTool {+ fn def(&self) -> &ToolDef { Self::def_static() }+ async fn invoke(&self, args: Value) -> ToolResult {+ let enabled = args.get("enabled").and_then(|v| v.as_bool()).unwrap_or(true);+ let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default");+ crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled));+ ToolResult::text(format!("Agent cursor '{}' {}.", cursor_id, if enabled { "enabled" } else { "disabled" }))+ }+ }+}+pub use set_enabled_m::SetAgentCursorEnabledTool;++pub fn build_registry() -> mcp_server::tool::ToolRegistry {+ let mut r = mcp_server::tool::ToolRegistry::new();+ r.register(Box::new(ListAppsTool));+ r.register(Box::new(ListWindowsTool));+ r.register(Box::new(GetWindowStateTool));+ r.register(Box::new(LaunchAppTool));+ r.register(Box::new(ClickTool));+ r.register(Box::new(DoubleClickTool));+ r.register(Box::new(RightClickTool));+ r.register(Box::new(TypeTextTool));+ r.register(Box::new(PressKeyTool));+ r.register(Box::new(HotkeyTool));+ r.register(Box::new(SetValueTool));+ r.register(Box::new(ScrollTool));+ r.register(Box::new(ScreenshotTool));+ r.register(Box::new(GetScreenSizeTool));+ r.register(Box::new(GetCursorPositionTool));+ r.register(Box::new(MoveCursorTool));+ r.register(Box::new(SetAgentCursorEnabledTool));+ r.register(Box::new(SetAgentCursorMotionTool));+ r.register(Box::new(GetAgentCursorStateTool));+ r.register(Box::new(SetAgentCursorStyleTool));+ r.register(Box::new(CheckPermissionsTool));+ r.register(Box::new(GetConfigTool));+ r.register(Box::new(SetConfigTool));+ r.register(Box::new(GetAccessibilityTreeTool));+ r.register(Box::new(ZoomTool));+ r.register(Box::new(TypeTextCharsTool));+ r.register_recording_tools();+ r+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/uia/cache.rs b/libs/cua-driver-rs/crates/platform-windows/src/uia/cache.rsnew file mode 100644index 0000000000..50056d1a42--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/uia/cache.rs@@ -0,0 +1,78 @@+//! UIA element cache for Windows.+//!+//! Mirrors the macOS ElementCache design: per-(pid, window_id) snapshots of+//! retained IUIAutomationElement COM pointers.+//!+//! Memory contract: UiaNode::element_ptr is a raw IUIAutomationElement vtable+//! pointer with an extra AddRef from clone()+forget() in the walker. Drop here+//! calls Release() to balance.++use super::UiaNode;+use std::collections::HashMap;+use std::sync::Mutex;+use windows::Win32::UI::Accessibility::IUIAutomationElement;+use windows::core::Interface;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct CacheKey {+ pub pid: u32,+ pub hwnd: u64,+}++pub struct CachedSnapshot {+ /// element_index → raw IUIAutomationElement pointer (retained).+ pub elements: Vec<usize>,+ /// element_index → screen-coordinate center (captured at walk time).+ pub centers: Vec<(i32, i32)>,+}++impl Drop for CachedSnapshot {+ fn drop(&mut self) {+ for ptr in &self.elements {+ if *ptr != 0 {+ // Reconstruct an interface reference and drop it (calls Release).+ unsafe {+ let iface: IUIAutomationElement = IUIAutomationElement::from_raw(*ptr as *mut _);+ drop(iface);+ }+ }+ }+ }+}++pub struct ElementCache {+ inner: Mutex<HashMap<CacheKey, CachedSnapshot>>,+}++impl ElementCache {+ pub fn new() -> Self {+ Self { inner: Mutex::new(HashMap::new()) }+ }++ pub fn update(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) {+ let actionable: Vec<&UiaNode> = nodes.iter().filter(|n| n.element_index.is_some()).collect();+ let elements: Vec<usize> = actionable.iter().map(|n| n.element_ptr).collect();+ let centers: Vec<(i32, i32)> = actionable.iter().map(|n| (n.center_x, n.center_y)).collect();+ let mut inner = self.inner.lock().unwrap();+ inner.insert(CacheKey { pid, hwnd }, CachedSnapshot { elements, centers });+ }++ pub fn get_element_ptr(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<usize> {+ let inner = self.inner.lock().unwrap();+ inner.get(&CacheKey { pid, hwnd })?.elements.get(element_index).copied()+ }++ pub fn get_element_center(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<(i32, i32)> {+ let inner = self.inner.lock().unwrap();+ inner.get(&CacheKey { pid, hwnd })?.centers.get(element_index).copied()+ }++ pub fn element_count(&self, pid: u32, hwnd: u64) -> usize {+ let inner = self.inner.lock().unwrap();+ inner.get(&CacheKey { pid, hwnd }).map(|s| s.elements.len()).unwrap_or(0)+ }+}++impl Default for ElementCache {+ fn default() -> Self { Self::new() }+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rsnew file mode 100644index 0000000000..6b3c931c57--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rs@@ -0,0 +1,412 @@+//! UI Automation (UIA) tree walking for Windows.+//!+//! Produces the same Markdown format as the macOS AX tree:+//! `INDENT- [N] ControlType "Name" [value="..." id=... actions=[...]]`+//! `INDENT- ControlType = "Value"` (non-indexed read-only elements)+//!+//! Uses IUIAutomation COM interface (available Windows 7+).+//! Uses IUIAutomationCacheRequest to batch-fetch all properties in one RPC+//! (avoids per-property cross-process calls that make Chrome's 5000-node tree+//! take >4s when reading each property individually).++use windows::core::{Interface, BSTR};+use windows::Win32::System::Com::{CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED};+use windows::Win32::UI::Accessibility::{+ CUIAutomation, IUIAutomation, IUIAutomationCacheRequest, IUIAutomationElement,+ UIA_AutomationIdPropertyId, UIA_BoundingRectanglePropertyId,+ UIA_ControlTypePropertyId, UIA_HelpTextPropertyId,+ UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_NamePropertyId,+ UIA_ValueValuePropertyId, UIA_InvokePatternId, UIA_SelectionItemPatternId,+ UIA_TogglePatternId, UIA_ExpandCollapsePatternId, UIA_TextPatternId,+ UIA_ValuePatternId, UIA_ScrollPatternId,+ TreeScope_Subtree,+};++pub mod cache;+pub use cache::ElementCache;++const MAX_DEPTH: usize = 25;+const MAX_TOTAL_ELEMENTS: usize = 5000;++/// A single node in the UIA tree.+#[derive(Clone)]+pub struct UiaNode {+ pub element_index: Option<usize>,+ pub control_type: String,+ pub name: Option<String>,+ pub value: Option<String>,+ pub automation_id: Option<String>,+ pub help_text: Option<String>,+ pub actions: Vec<String>,+ /// Raw IUIAutomationElement pointer as usize (retained for cache).+ pub element_ptr: usize,+ /// Screen-coordinate center, captured at walk time to avoid later COM calls.+ pub center_x: i32,+ pub center_y: i32,+}++pub struct UiaTreeResult {+ pub tree_markdown: String,+ pub nodes: Vec<UiaNode>,+}++/// Walk the UIA tree for the window with the given HWND.+pub fn walk_tree(hwnd: u64, query: Option<&str>) -> UiaTreeResult {+ unsafe { walk_tree_unsafe(hwnd, query) }+}++unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {+ let _ = CoInitializeEx(None, COINIT_MULTITHREADED);++ let automation: IUIAutomation = match CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) {+ Ok(a) => a,+ Err(e) => return UiaTreeResult {+ tree_markdown: format!("UIA init failed: {e}"),+ nodes: Vec::new(),+ },+ };++ // Build a cache request that fetches everything we need in ONE bulk RPC.+ let cache_req: IUIAutomationCacheRequest = match automation.CreateCacheRequest() {+ Ok(r) => r,+ Err(e) => return UiaTreeResult {+ tree_markdown: format!("CreateCacheRequest failed: {e}"),+ nodes: Vec::new(),+ },+ };++ // Properties to pre-fetch.+ for prop in &[+ UIA_ControlTypePropertyId,+ UIA_NamePropertyId,+ UIA_ValueValuePropertyId,+ UIA_AutomationIdPropertyId,+ UIA_HelpTextPropertyId,+ UIA_IsEnabledPropertyId,+ UIA_IsOffscreenPropertyId,+ UIA_BoundingRectanglePropertyId,+ ] {+ let _ = cache_req.AddProperty(*prop);+ }++ // Patterns to pre-fetch (for action detection).+ for pat in &[+ UIA_InvokePatternId,+ UIA_TogglePatternId,+ UIA_SelectionItemPatternId,+ UIA_ExpandCollapsePatternId,+ UIA_ValuePatternId,+ UIA_TextPatternId,+ UIA_ScrollPatternId,+ ] {+ let _ = cache_req.AddPattern(*pat);+ }++ // Fetch entire subtree in one call.+ let _ = cache_req.SetTreeScope(TreeScope_Subtree);++ // Apply control-view filter (same as ControlViewWalker).+ if let Ok(ctrl_cond) = automation.ControlViewCondition() {+ let _ = cache_req.SetTreeFilter(&ctrl_cond);+ }++ let hwnd_win = windows::Win32::Foundation::HWND(hwnd as *mut _);+ let root_elem = match automation.ElementFromHandleBuildCache(hwnd_win, &cache_req) {+ Ok(e) => e,+ Err(e) => return UiaTreeResult {+ tree_markdown: format!("ElementFromHandleBuildCache failed: {e}"),+ nodes: Vec::new(),+ },+ };++ let mut nodes: Vec<UiaNode> = Vec::new();+ let mut lines: Vec<(usize, String)> = Vec::new();+ let mut counter = 0usize;+ let mut total = 0usize;++ walk_cached(&root_elem, 0, &mut nodes, &mut lines, &mut counter, &mut total);++ let raw_md = render_lines(&lines);+ let tree_markdown = if let Some(q) = query {+ filter_tree(&raw_md, q)+ } else {+ raw_md+ };++ UiaTreeResult { tree_markdown, nodes }+}++unsafe fn walk_cached(+ element: &IUIAutomationElement,+ depth: usize,+ nodes: &mut Vec<UiaNode>,+ lines: &mut Vec<(usize, String)>,+ counter: &mut usize,+ total: &mut usize,+) {+ if depth > MAX_DEPTH || *total >= MAX_TOTAL_ELEMENTS {+ return;+ }+ *total += 1;++ let control_type = read_cached_control_type(element);+ let name = read_cached_bstr_name(element);+ let value = read_cached_bstr_value(element);+ let automation_id = read_cached_bstr(element, UIA_AutomationIdPropertyId);+ let help_text = read_cached_bstr(element, UIA_HelpTextPropertyId);+ let is_enabled = read_cached_bool(element, UIA_IsEnabledPropertyId).unwrap_or(true);+ let is_offscreen = read_cached_bool(element, UIA_IsOffscreenPropertyId).unwrap_or(false);++ let actions = detect_cached_actions(element, is_enabled);+ let is_actionable = !actions.is_empty() && is_enabled && !is_offscreen;+ let has_content = name.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false)+ || value.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false);++ if is_actionable || has_content {+ let retained: IUIAutomationElement = element.clone();+ let ptr = retained.as_raw() as usize;+ std::mem::forget(retained);++ let node = if is_actionable {+ let idx = *counter;+ *counter += 1;+ let (center_x, center_y) = read_cached_bounding_rect(element);+ UiaNode {+ element_index: Some(idx),+ control_type: control_type.clone(),+ name: name.clone(),+ value: value.clone(),+ automation_id: automation_id.clone(),+ help_text: help_text.clone(),+ actions: actions.clone(),+ element_ptr: ptr,+ center_x,+ center_y,+ }+ } else {+ UiaNode {+ element_index: None,+ control_type: control_type.clone(),+ name: name.clone(),+ value: value.clone(),+ automation_id: automation_id.clone(),+ help_text: help_text.clone(),+ actions: vec![],+ element_ptr: ptr,+ center_x: 0,+ center_y: 0,+ }+ };++ lines.push((depth, format_node_line(&node)));+ nodes.push(node);+ }++ // Recurse using cached children (no additional RPC).+ if let Ok(children) = element.GetCachedChildren() {+ let len = children.Length().unwrap_or(0);+ for i in 0..len {+ if let Ok(child) = children.GetElement(i) {+ walk_cached(&child, depth + 1, nodes, lines, counter, total);+ }+ }+ }+}++fn read_cached_control_type(element: &IUIAutomationElement) -> String {+ unsafe {+ element.CachedControlType().ok()+ .map(|ct| control_type_name(ct.0))+ .unwrap_or_else(|| "Unknown".into())+ }+}++fn read_cached_bstr_name(element: &IUIAutomationElement) -> Option<String> {+ unsafe {+ let bstr = element.CachedName().ok()?;+ let s = bstr.to_string();+ if s.trim().is_empty() { None } else { Some(s) }+ }+}++fn read_cached_bstr_value(element: &IUIAutomationElement) -> Option<String> {+ read_cached_bstr(element, UIA_ValueValuePropertyId)+}++fn read_cached_bstr(element: &IUIAutomationElement, property_id: windows::Win32::UI::Accessibility::UIA_PROPERTY_ID) -> Option<String> {+ unsafe {+ let variant = element.GetCachedPropertyValue(property_id).ok()?;+ if variant.as_raw().Anonymous.Anonymous.vt == 8 {+ let bstr = BSTR::from_raw(variant.as_raw().Anonymous.Anonymous.Anonymous.bstrVal);+ let s = bstr.to_string();+ std::mem::forget(bstr);+ if s.trim().is_empty() { None } else { Some(s) }+ } else {+ None+ }+ }+}++fn read_cached_bool(element: &IUIAutomationElement, property_id: windows::Win32::UI::Accessibility::UIA_PROPERTY_ID) -> Option<bool> {+ unsafe {+ let variant = element.GetCachedPropertyValue(property_id).ok()?;+ if variant.as_raw().Anonymous.Anonymous.vt == 11 {+ Some(variant.as_raw().Anonymous.Anonymous.Anonymous.boolVal != 0)+ } else {+ None+ }+ }+}++fn read_cached_bounding_rect(element: &IUIAutomationElement) -> (i32, i32) {+ unsafe {+ element.CachedBoundingRectangle()+ .map(|r| ((r.left + r.right) / 2, (r.top + r.bottom) / 2))+ .unwrap_or((0, 0))+ }+}++fn detect_cached_actions(element: &IUIAutomationElement, is_enabled: bool) -> Vec<String> {+ if !is_enabled { return vec![]; }+ let mut actions = Vec::new();+ unsafe {+ if element.GetCachedPattern(UIA_InvokePatternId).is_ok() {+ actions.push("invoke".into());+ }+ if element.GetCachedPattern(UIA_TogglePatternId).is_ok() {+ actions.push("toggle".into());+ }+ if element.GetCachedPattern(UIA_SelectionItemPatternId).is_ok() {+ actions.push("select".into());+ }+ if element.GetCachedPattern(UIA_ExpandCollapsePatternId).is_ok() {+ actions.push("expand".into());+ }+ if element.GetCachedPattern(UIA_ValuePatternId).is_ok() {+ actions.push("set_value".into());+ }+ if element.GetCachedPattern(UIA_TextPatternId).is_ok() {+ actions.push("text".into());+ }+ if element.GetCachedPattern(UIA_ScrollPatternId).is_ok() {+ actions.push("scroll".into());+ }+ }+ actions+}++fn control_type_name(id: i32) -> String {+ match id {+ 50000 => "Button",+ 50001 => "Calendar",+ 50002 => "CheckBox",+ 50003 => "ComboBox",+ 50004 => "Edit",+ 50005 => "Hyperlink",+ 50006 => "Image",+ 50007 => "ListItem",+ 50008 => "List",+ 50009 => "Menu",+ 50010 => "MenuBar",+ 50011 => "MenuItem",+ 50012 => "ProgressBar",+ 50013 => "RadioButton",+ 50014 => "ScrollBar",+ 50015 => "Slider",+ 50016 => "Spinner",+ 50017 => "StatusBar",+ 50018 => "Tab",+ 50019 => "TabItem",+ 50020 => "Text",+ 50021 => "ToolBar",+ 50022 => "ToolTip",+ 50023 => "Tree",+ 50024 => "TreeItem",+ 50025 => "Custom",+ 50026 => "Group",+ 50027 => "Thumb",+ 50028 => "DataGrid",+ 50029 => "DataItem",+ 50030 => "Document",+ 50031 => "SplitButton",+ 50032 => "Window",+ 50033 => "Pane",+ 50034 => "Header",+ 50035 => "HeaderItem",+ 50036 => "Table",+ 50037 => "TitleBar",+ 50038 => "Separator",+ 50039 => "SemanticZoom",+ 50040 => "AppBar",+ _ => "Unknown",+ }.into()+}++fn format_node_line(node: &UiaNode) -> String {+ let mut s = String::new();+ if let Some(idx) = node.element_index {+ s.push_str(&format!("- [{}] {}", idx, node.control_type));+ if let Some(n) = &node.name {+ s.push_str(&format!(" \"{}\"", n));+ }+ let mut attrs = Vec::new();+ if let Some(v) = &node.value { attrs.push(format!("value=\"{}\"", v)); }+ if let Some(id) = &node.automation_id { attrs.push(format!("id={}", id)); }+ if let Some(h) = &node.help_text { attrs.push(format!("help=\"{}\"", h)); }+ if !node.actions.is_empty() {+ attrs.push(format!("actions=[{}]", node.actions.join(",")));+ }+ if !attrs.is_empty() {+ s.push_str(&format!(" [{}]", attrs.join(" ")));+ }+ } else {+ s.push_str(&format!("- {}", node.control_type));+ if let Some(n) = &node.name { s.push_str(&format!(" \"{}\"", n)); }+ if let Some(v) = &node.value { s.push_str(&format!(" = \"{}\"", v)); }+ }+ s+}++fn render_lines(lines: &[(usize, String)]) -> String {+ let mut out = String::new();+ for (depth, line) in lines {+ for _ in 0..*depth { out.push_str(" "); }+ out.push_str(line);+ out.push('\n');+ }+ out+}++fn filter_tree(markdown: &str, query: &str) -> String {+ let needle = query.to_lowercase();+ let lines: Vec<&str> = markdown.lines().collect();+ let mut ancestors: Vec<&str> = Vec::new();+ let mut last_emitted: Vec<Option<&str>> = Vec::new();+ let mut output: Vec<&str> = Vec::new();++ for line in &lines {+ let depth = line.chars().take_while(|c| *c == ' ').count() / 2;+ while ancestors.len() <= depth {+ ancestors.push("");+ last_emitted.push(None);+ }+ for d in (depth + 1)..ancestors.len() { last_emitted[d] = None; }+ ancestors[depth] = line;++ if line.to_lowercase().contains(&needle) {+ for d in 0..depth {+ if ancestors[d].is_empty() { continue; }+ if last_emitted[d] == Some(ancestors[d]) { continue; }+ last_emitted[d] = Some(ancestors[d]);+ output.push(ancestors[d]);+ }+ last_emitted[depth] = Some(line);+ output.push(line);+ }+ }++ if output.is_empty() { return String::new(); }+ let mut r = output.join("\n");+ r.push('\n');+ r+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rsnew file mode 100644index 0000000000..9e81d35568--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs@@ -0,0 +1,54 @@+//! Enumerate running processes on Windows.+//!+//! Uses CreateToolhelp32Snapshot / Process32FirstW / Process32NextW (simpler+//! and more portable than NtQuerySystemInformation or EnumProcesses+psapi).++use windows::Win32::Foundation::CloseHandle;+use windows::Win32::System::Diagnostics::ToolHelp::{+ CreateToolhelp32Snapshot, Process32FirstW, Process32NextW, PROCESSENTRY32W,+ TH32CS_SNAPPROCESS,+};++#[derive(Debug, Clone)]+pub struct ProcessInfo {+ pub pid: u32,+ pub parent_pid: u32,+ pub name: String,+}++/// Return all running processes (pid, parent_pid, executable name).+pub fn list_processes() -> Vec<ProcessInfo> {+ let mut result = Vec::new();+ unsafe {+ let snap = match CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) {+ Ok(h) => h,+ Err(_) => return result,+ };++ let mut entry = PROCESSENTRY32W {+ dwSize: std::mem::size_of::<PROCESSENTRY32W>() as u32,+ ..Default::default()+ };++ if Process32FirstW(snap, &mut entry).is_ok() {+ loop {+ let name = decode_wstr(&entry.szExeFile);+ result.push(ProcessInfo {+ pid: entry.th32ProcessID,+ parent_pid: entry.th32ParentProcessID,+ name,+ });+ if Process32NextW(snap, &mut entry).is_err() {+ break;+ }+ }+ }+ let _ = CloseHandle(snap);+ }+ result+}++fn decode_wstr(buf: &[u16]) -> String {+ let len = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());+ String::from_utf16_lossy(&buf[..len])+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rsnew file mode 100644index 0000000000..704682a1e6--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs@@ -0,0 +1,7 @@+//! Win32 API wrappers for window/process enumeration.++pub mod apps;+pub mod windows;++pub use apps::{list_processes, ProcessInfo};+pub use windows::{list_windows, WindowInfo};diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/windows.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/windows.rsnew file mode 100644index 0000000000..4c9c161e8c--- /dev/null+++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/windows.rs@@ -0,0 +1,107 @@+//! Enumerate top-level windows on Windows.+//!+//! Uses EnumWindows + GetWindowTextW + GetWindowThreadProcessId.+//! Returns only visible, non-minimized windows with a non-empty title.++use std::sync::Mutex;+use windows::Win32::Foundation::{BOOL, HWND, LPARAM, RECT, TRUE};+use windows::Win32::Graphics::Dwm::{+ DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS,+};+use windows::Win32::UI::WindowsAndMessaging::{+ EnumWindows, GetWindowRect, GetWindowTextLengthW, GetWindowTextW,+ GetWindowThreadProcessId, IsIconic, IsWindowVisible,+};++#[derive(Debug, Clone)]+pub struct WindowInfo {+ /// HWND cast to u64 for serialization.+ pub hwnd: u64,+ /// pid owning the window.+ pub pid: u32,+ pub title: String,+ pub x: i32,+ pub y: i32,+ pub width: i32,+ pub height: i32,+}++struct EnumState {+ filter_pid: Option<u32>,+ windows: Vec<WindowInfo>,+}++/// List top-level visible windows. If `filter_pid` is Some, only that process.+pub fn list_windows(filter_pid: Option<u32>) -> Vec<WindowInfo> {+ let state = Mutex::new(EnumState { filter_pid, windows: Vec::new() });+ let state_ptr = &state as *const Mutex<EnumState> as isize;+ unsafe {+ let _ = EnumWindows(Some(enum_windows_cb), LPARAM(state_ptr));+ }+ state.into_inner().unwrap().windows+}++unsafe extern "system" fn enum_windows_cb(hwnd: HWND, lparam: LPARAM) -> BOOL {+ let state = &*(lparam.0 as *const Mutex<EnumState>);++ // Skip invisible or minimized windows.+ if IsWindowVisible(hwnd).0 == 0 || IsIconic(hwnd).0 != 0 {+ return TRUE;+ }++ // Get pid.+ let mut pid: u32 = 0;+ GetWindowThreadProcessId(hwnd, Some(&mut pid));++ // Filter by pid if requested.+ {+ let s = state.lock().unwrap();+ if let Some(fp) = s.filter_pid {+ if pid != fp { return TRUE; }+ }+ }++ // Get title (skip empty).+ let title_len = GetWindowTextLengthW(hwnd);+ if title_len == 0 { return TRUE; }+ let mut buf = vec![0u16; (title_len + 1) as usize];+ GetWindowTextW(hwnd, &mut buf);+ let title = {+ let len = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());+ String::from_utf16_lossy(&buf[..len])+ };+ if title.trim().is_empty() { return TRUE; }++ // Get bounds — prefer DWM extended frame bounds (includes shadow), fallback to GetWindowRect.+ let (x, y, w, h) = get_window_bounds(hwnd);++ state.lock().unwrap().windows.push(WindowInfo {+ hwnd: hwnd.0 as u64,+ pid,+ title,+ x,+ y,+ width: w,+ height: h,+ });++ TRUE+}++fn get_window_bounds(hwnd: HWND) -> (i32, i32, i32, i32) {+ unsafe {+ let mut rect = RECT::default();+ // Try DwmGetWindowAttribute for accurate bounds (excludes drop shadow on W11).+ let ok = DwmGetWindowAttribute(+ hwnd,+ DWMWA_EXTENDED_FRAME_BOUNDS,+ &mut rect as *mut RECT as *mut _,+ std::mem::size_of::<RECT>() as u32,+ );+ if ok.is_err() {+ // Fallback to GetWindowRect.+ let _ = GetWindowRect(hwnd, &mut rect);+ }+ (rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top)+ }+}diff --git a/libs/cua-driver-rs/sandbox/run-tests-in-sandbox.ps1 b/libs/cua-driver-rs/sandbox/run-tests-in-sandbox.ps1new file mode 100644index 0000000000..f8b07a2404--- /dev/null+++ b/libs/cua-driver-rs/sandbox/run-tests-in-sandbox.ps1@@ -0,0 +1,190 @@+# run-tests-in-sandbox.ps1 — host-side launcher with live log streaming+#+# Usage (from workspace root):+# .\sandbox\run-tests-in-sandbox.ps1 [<test-filter>]+#+# Steps:+# 1. cargo build + cargo test --no-run+# 2. Launch Windows Sandbox with workspace mapped read-only+# 3. Tail runner-log.txt and test-output.txt live (2s poll)+# 4. Exit with sandbox test exit code++param([string]$TestFilter = "")++Set-StrictMode -Version Latest+$ErrorActionPreference = "Stop"++function Close-Sandbox {+ Write-Host "`n[SANDBOX] Closing sandbox..." -ForegroundColor Yellow+ $names = @("WindowsSandboxClient", "WindowsSandbox")+ foreach ($n in $names) {+ $procs = Get-Process $n -ErrorAction SilentlyContinue+ if ($procs) {+ Write-Host " Stopping $n (pid $($procs.Id -join ','))..."+ $procs | Stop-Process -Force -ErrorAction SilentlyContinue+ }+ }+ Start-Sleep -Milliseconds 1500+ $ErrorActionPreference = "Continue"+ foreach ($n in $names) { & taskkill /F /IM "$n.exe" }+ $ErrorActionPreference = "Stop"+ Write-Host " Sandbox closed." -ForegroundColor Green+}++$sandboxDir = Split-Path -Parent $MyInvocation.MyCommand.Definition+$wsRoot = Split-Path -Parent $sandboxDir++Write-Host "=== cua-driver-rs Windows Sandbox Test Runner ===" -ForegroundColor Cyan+Write-Host "Workspace: $wsRoot"++# ── Pre-flight: fail fast if a sandbox is already running ────────────────────+$running = Get-Process "WindowsSandboxClient","WindowsSandbox" -ErrorAction SilentlyContinue+if ($running) {+ Write-Host "`n[ERROR] Windows Sandbox is already running (pid $($running.Id -join ','))." -ForegroundColor Red+ Write-Host " Only one sandbox instance is allowed at a time." -ForegroundColor Red+ Write-Host " Close it first, or run: Close-Sandbox" -ForegroundColor Yellow+ Write-Host " Killing existing sandbox and retrying..." -ForegroundColor Yellow+ Close-Sandbox+ $running = Get-Process "WindowsSandboxClient","WindowsSandbox" -ErrorAction SilentlyContinue+ if ($running) {+ Write-Host "[ERROR] Could not kill existing sandbox. Aborting." -ForegroundColor Red+ exit 1+ }+ Write-Host "[OK] Existing sandbox closed. Proceeding." -ForegroundColor Green+}++# ── 0. Download test-app binaries if missing ─────────────────────────────────+$testAppExe = "$wsRoot\test-apps\desktop-test-app-electron.0.1.0.exe"+if (-not (Test-Path $testAppExe)) {+ Write-Host "`n[FETCH] Downloading desktop-test-app-electron..." -ForegroundColor Yellow+ New-Item -ItemType Directory -Force "$wsRoot\test-apps" | Out-Null+ $url = "https://github.com/trycua/desktop-test-app-electron/releases/download/v0.1.0/desktop-test-app-electron.0.1.0.exe"+ Invoke-WebRequest -Uri $url -OutFile $testAppExe -UseBasicParsing+ Write-Host " Downloaded: $testAppExe" -ForegroundColor Green+} else {+ Write-Host "`n[FETCH] test-app already present: $testAppExe" -ForegroundColor Green+}++# ── 1. Build ──────────────────────────────────────────────────────────────────+Push-Location $wsRoot+try {+ Write-Host "`n[BUILD] cargo build..." -ForegroundColor Yellow+ cargo build+ if ($LASTEXITCODE -ne 0) { throw "cargo build failed" }++ Write-Host "`n[BUILD] cargo test --no-run (mcp_protocol_test)..." -ForegroundColor Yellow+ cargo test --test mcp_protocol_test --no-run+ if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (mcp_protocol_test) failed" }++ Write-Host "`n[BUILD] cargo test --no-run (ux_guard_test)..." -ForegroundColor Yellow+ cargo test --test ux_guard_test --no-run+ if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (ux_guard_test) failed" }+} finally { Pop-Location }++$mcpBin = Get-ChildItem "$wsRoot\target\debug\deps\mcp_protocol_test-*.exe" |+ Sort-Object LastWriteTime -Descending | Select-Object -First 1+if (-not $mcpBin) { throw "mcp_protocol_test-*.exe not found" }+Write-Host "mcp_protocol_test: $($mcpBin.Name)"++$uxBin = Get-ChildItem "$wsRoot\target\debug\deps\ux_guard_test-*.exe" |+ Sort-Object LastWriteTime -Descending | Select-Object -First 1+if ($uxBin) { Write-Host "ux_guard_test : $($uxBin.Name)" } else { Write-Host "ux_guard_test : (not found, will skip)" }++# ── 2. Prepare shared output folder ──────────────────────────────────────────+$outputDir = "$env:TEMP\cua-sandbox-output"+New-Item -ItemType Directory -Force $outputDir | Out-Null+Get-ChildItem $outputDir -ErrorAction SilentlyContinue | Remove-Item -Force -Recurse -ErrorAction SilentlyContinue+$TestFilter | Out-File "$outputDir\test-filter.txt" -Encoding utf8 -NoNewline++# ── 3. Generate .wsb and launch ───────────────────────────────────────────────+$wsbContent = @"+<Configuration>+ <MappedFolders>+ <MappedFolder>+ <HostFolder>$wsRoot</HostFolder>+ <SandboxFolder>C:\cua-driver-rs</SandboxFolder>+ <ReadOnly>true</ReadOnly>+ </MappedFolder>+ <MappedFolder>+ <HostFolder>$outputDir</HostFolder>+ <SandboxFolder>C:\sandbox-output</SandboxFolder>+ <ReadOnly>false</ReadOnly>+ </MappedFolder>+ </MappedFolders>+ <LogonCommand>+ <Command>powershell -ExecutionPolicy Bypass -NonInteractive -File C:\cua-driver-rs\sandbox\sandbox-runner.ps1</Command>+ </LogonCommand>+</Configuration>+"@+$wsbPath = "$env:TEMP\cua-driver-test.wsb"+$wsbContent | Out-File $wsbPath -Encoding utf8++Write-Host "`n[SANDBOX] Launching Windows Sandbox..." -ForegroundColor Yellow+$preLaunchCheck = Get-Process "WindowsSandboxClient","WindowsSandbox" -ErrorAction SilentlyContinue+if ($preLaunchCheck) {+ Write-Host "[ERROR] A sandbox appeared just before launch. Aborting." -ForegroundColor Red+ exit 1+}+Start-Process $wsbPath++# ── 4. Stream logs live ────────────────────────────────────────────────────────+Write-Host "[STREAM] Tailing logs (Ctrl-C to abort)..." -ForegroundColor Yellow++$doneFile = "$outputDir\done.txt"+$logFile = "$outputDir\runner-log.txt"+$outputFile = "$outputDir\test-output.txt"+$logPos = 0+$outPos = 0+$timeout = 600+$elapsed = 0+$pollInterval = 2++while ($elapsed -lt $timeout) {+ if (Test-Path $logFile) {+ $content = Get-Content $logFile -Raw -Encoding utf8 -ErrorAction SilentlyContinue+ if ($content -and $content.Length -gt $logPos) {+ $newText = $content.Substring($logPos)+ $logPos = $content.Length+ $newText.TrimEnd() -split "`n" | ForEach-Object { Write-Host "[sandbox] $_" -ForegroundColor DarkCyan }+ }+ }+ if (Test-Path $outputFile) {+ $content = Get-Content $outputFile -Raw -Encoding utf8 -ErrorAction SilentlyContinue+ if ($content -and $content.Length -gt $outPos) {+ $newText = $content.Substring($outPos)+ $outPos = $content.Length+ $newText.TrimEnd() -split "`n" | ForEach-Object { Write-Host " $_" }+ }+ }+ if (Test-Path $doneFile) { break }+ Start-Sleep -Seconds $pollInterval+ $elapsed += $pollInterval+}++# Final drain+foreach ($f in @($logFile, $outputFile)) {+ if (Test-Path $f) {+ $content = Get-Content $f -Raw -Encoding utf8 -ErrorAction SilentlyContinue+ $pos = if ($f -eq $logFile) { $logPos } else { $outPos }+ if ($content -and $content.Length -gt $pos) {+ $content.Substring($pos).TrimEnd() -split "`n" | ForEach-Object { Write-Host " $_" }+ }+ }+}++if (-not (Test-Path $doneFile)) {+ Write-Host "`nTIMEOUT after ${elapsed}s." -ForegroundColor Red+ Close-Sandbox+ exit 1+}++$exitCode = 0+if (Test-Path "$outputDir\exit-code.txt") {+ $exitCode = [int](Get-Content "$outputDir\exit-code.txt" -Raw).Trim()+}++Write-Host ""+if ($exitCode -eq 0) { Write-Host "PASSED (exit 0)" -ForegroundColor Green } else { Write-Host "FAILED (exit $exitCode)" -ForegroundColor Red }++Close-Sandbox+exit $exitCodediff --git a/libs/cua-driver-rs/sandbox/sandbox-runner.ps1 b/libs/cua-driver-rs/sandbox/sandbox-runner.ps1new file mode 100644index 0000000000..951e82038c--- /dev/null+++ b/libs/cua-driver-rs/sandbox/sandbox-runner.ps1@@ -0,0 +1,114 @@+# sandbox-runner.ps1 — runs INSIDE Windows Sandbox as LogonCommand+# Streams test output live to C:\sandbox-output\ so the host can tail it.++$ErrorActionPreference = "Continue"+$logFile = "C:\sandbox-output\runner-log.txt"+$outputFile = "C:\sandbox-output\test-output.txt"+$exitFile = "C:\sandbox-output\exit-code.txt"+$doneFile = "C:\sandbox-output\done.txt"++function Log($msg) {+ $ts = (Get-Date).ToString("HH:mm:ss")+ "$ts $msg" | Tee-Object -FilePath $logFile -Append | Write-Host+}++Log "sandbox-runner.ps1 starting"++# ── set env so binary_path() resolves cua-driver.exe correctly ─────────────+$env:CARGO_MANIFEST_DIR = "C:\cua-driver-rs\crates\cua-driver"+Log "CARGO_MANIFEST_DIR = $env:CARGO_MANIFEST_DIR"++# ── copy test-app exe to %TEMP% so ShellExecuteW doesn't hit a zone dialog ──+# Files on Sandbox-mapped folders are Zone 3 (Internet); ShellExecuteW blocks+# on the "Do you want to run this file?" dialog. A local %TEMP% copy is Zone 1.+$testAppSrc = "C:\cua-driver-rs\test-apps\desktop-test-app-electron.0.1.0.exe"+$testAppDst = "$env:TEMP\desktop-test-app-electron.0.1.0.exe"+if (Test-Path $testAppSrc) {+ Copy-Item $testAppSrc $testAppDst -Force+ $env:TEST_APP_EXE = $testAppDst+ Log "test-app copied to $testAppDst (TEST_APP_EXE=$testAppDst)"+} else {+ Log "WARNING: test-app not found at $testAppSrc"+}++$driverExe = "C:\cua-driver-rs\target\debug\cua-driver.exe"+if (-not (Test-Path $driverExe)) {+ Log "ERROR: $driverExe not found"+ "1" | Out-File $exitFile -Encoding utf8 -NoNewline+ "done" | Out-File $doneFile -Encoding utf8 -NoNewline+ exit 1+}+Log "cua-driver : $driverExe"++# ── find test binaries ───────────────────────────────────────────────────────+# Run mcp_protocol_test first, then ux_guard_test (UX guard needs a real+# desktop session and spawns visible windows, so it runs second).+$testSuites = @(+ @{ Pattern = "mcp_protocol_test-*.exe"; Label = "mcp_protocol_test" },+ @{ Pattern = "ux_guard_test-*.exe"; Label = "ux_guard_test" }+)++# ── optional test filter ────────────────────────────────────────────────────+$filterSuite = ""+if (Test-Path "C:\sandbox-output\test-filter.txt") {+ $filterSuite = (Get-Content "C:\sandbox-output\test-filter.txt" -Raw).Trim()+}+$filterLabel = if ($filterSuite) { $filterSuite } else { "(all)" }+Log "Suite filter : $filterLabel"++"" | Out-File $outputFile -Encoding utf8 # create file so host can start tailing++$overallExit = 0++foreach ($suite in $testSuites) {+ $label = $suite.Label+ # If a filter was given and this suite's label doesn't match, skip it.+ if ($filterSuite -ne "" -and $label -notlike "*$filterSuite*") {+ Log "Skipping $label (filter='$filterSuite')"+ continue+ }++ $testBin = Get-ChildItem "C:\cua-driver-rs\target\debug\deps\$($suite.Pattern)" |+ Sort-Object LastWriteTime -Descending | Select-Object -First 1+ if (-not $testBin) {+ Log "WARNING: $($suite.Pattern) not found, skipping $label"+ continue+ }+ Log "Running $label : $($testBin.FullName)"++ $testArgs = @("--test-threads=1", "--nocapture")+ $stderrFile = "C:\sandbox-output\$label-stderr.txt"++ "`n=== $label ===`n" | Add-Content $outputFile -Encoding utf8++ $proc = Start-Process `+ -FilePath $testBin.FullName `+ -ArgumentList $testArgs `+ -RedirectStandardOutput "C:\sandbox-output\$label-stdout.txt" `+ -RedirectStandardError $stderrFile `+ -NoNewWindow `+ -PassThru `+ -Wait++ # Append stdout to the shared output file.+ if (Test-Path "C:\sandbox-output\$label-stdout.txt") {+ Get-Content "C:\sandbox-output\$label-stdout.txt" -Raw |+ Add-Content $outputFile -Encoding utf8+ }+ # Append stderr.+ if (Test-Path $stderrFile) {+ $stderr = Get-Content $stderrFile -Raw+ if ($stderr.Trim() -ne "") {+ "`n--- $label stderr ---`n$stderr" | Add-Content $outputFile -Encoding utf8+ }+ }++ $suiteExit = $proc.ExitCode+ Log "$label exited : $suiteExit"+ if ($suiteExit -ne 0) { $overallExit = $suiteExit }+}++$overallExit.ToString() | Out-File $exitFile -Encoding utf8 -NoNewline+Log "Results written, writing done sentinel..."+"done" | Out-File $doneFile -Encoding utf8 -NoNewline+Log "Finished."diff --git a/libs/cua-driver-rs/scripts/install.sh b/libs/cua-driver-rs/scripts/install.shnew file mode 100644index 0000000000..4a35d2baec--- /dev/null+++ b/libs/cua-driver-rs/scripts/install.sh@@ -0,0 +1,161 @@+#!/usr/bin/env bash+# cua-driver-rs installer — download the latest cua-driver-rs release tarball+# from GitHub Releases and drop the binary into ~/.local/bin (or a path given+# via --bin-dir / CUA_DRIVER_RS_BIN_DIR). Sudo-free.+#+# This is the Rust port of cua-driver — cross-platform (macOS / Linux / Windows+# via WSL or git-bash) computer-use automation. The Swift cua-driver (macOS+# only) ships separately under tag prefix `cua-driver-v*` and is installed+# via `libs/cua-driver/scripts/install.sh`; this script is hard-pinned to+# `cua-driver-rs-v*` and will never pick up the Swift binary.+#+# Usage:+# /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver-rs/scripts/install.sh)"+#+# Flags:+# --bin-dir <path> install the binary to <path> instead of ~/.local/bin+# --no-modify-path skip auto-appending an `export PATH=...` line+#+# Env overrides:+# CUA_DRIVER_RS_VERSION=0.1.2 pin a specific release tag+# CUA_DRIVER_RS_BIN_DIR=PATH same as --bin-dir+# CUA_DRIVER_RS_NO_MODIFY_PATH=1 same as --no-modify-path+#+# ⚠️ This is a BETA release. The Rust port is feature-complete on Windows+# and Linux; macOS parity with the Swift cua-driver is in progress. For+# production macOS automation prefer `libs/cua-driver/scripts/install.sh`.+set -euo pipefail++REPO="trycua/cua"+BINARY_NAME="cua-driver"+TAG_PREFIX="cua-driver-rs-v"+BIN_DIR="${CUA_DRIVER_RS_BIN_DIR:-$HOME/.local/bin}"+NO_MODIFY_PATH="${CUA_DRIVER_RS_NO_MODIFY_PATH:-0}"++while [[ $# -gt 0 ]]; do+ case "$1" in+ --bin-dir) BIN_DIR="$2"; shift 2 ;;+ --bin-dir=*) BIN_DIR="${1#*=}"; shift ;;+ --no-modify-path) NO_MODIFY_PATH=1; shift ;;+ *) shift ;;+ esac+done++BIN_LINK="$BIN_DIR/$BINARY_NAME"+TMP_DIR=$(mktemp -d)+trap 'rm -rf "$TMP_DIR"' EXIT++log() { printf '==> %s\n' "$*"; }+err() { printf 'error: %s\n' "$*" >&2; }++# --- Resolve OS/arch ----------------------------------------------------++OS=$(uname -s)+ARCH_RAW=$(uname -m)+case "$OS-$ARCH_RAW" in+ Darwin-arm64|Darwin-aarch64) LABEL="darwin-arm64" ;;+ Darwin-x86_64) LABEL="darwin-x86_64" ;;+ Linux-x86_64|Linux-amd64) LABEL="linux-x86_64" ;;+ *)+ err "unsupported platform: $OS / $ARCH_RAW"+ err " cua-driver-rs ships prebuilts for: darwin-arm64, darwin-x86_64, linux-x86_64."+ err " Windows users should download cua-driver-rs-<v>-windows-x86_64.zip from GitHub Releases directly."+ exit 1+ ;;+esac++for cmd in curl tar; do+ if ! command -v "$cmd" >/dev/null 2>&1; then+ err "$cmd not found on PATH"+ exit 1+ fi+done++# --- Resolve release tag ------------------------------------------------++if [[ -n "${CUA_DRIVER_RS_VERSION:-}" ]]; then+ TAG="${TAG_PREFIX}${CUA_DRIVER_RS_VERSION#v}"+ log "using version from CUA_DRIVER_RS_VERSION: $TAG"+else+ log "resolving latest $TAG_PREFIX* release via GitHub API"+ # Pinned to the exact `cua-driver-rs-v*` prefix so this script can never+ # accidentally pick up a Swift `cua-driver-v*` release.+ TAG=$(curl -fsSL "https://api.github.com/repos/$REPO/releases?per_page=40" \+ | grep -Eo '"tag_name":[[:space:]]*"'"${TAG_PREFIX}"'[^"]+"' \+ | sed -E 's/.*"'"${TAG_PREFIX}"'([0-9]+[.][0-9]+[.][0-9]+)"/\1/' \+ | sort -t. -k1,1nr -k2,2nr -k3,3nr \+ | head -n 1 \+ | sed -E 's/^/'"${TAG_PREFIX}"'/')+ if [[ -z "$TAG" ]]; then+ err "no release matching ${TAG_PREFIX}* found on $REPO"+ err " (cua-driver-rs is a BETA-stage cross-platform port; releases may not be published yet.)"+ exit 1+ fi+ log "latest release: $TAG"+fi++VERSION="${TAG#${TAG_PREFIX}}"++# --- Download bare-binary tarball ---------------------------------------++# Prefer the bare-binary tarball (single `cua-driver` file at the root) —+# the directory tarball would require unpacking and copying, but the bare+# form is curl-pipe-able. Both forms are published per release.+TARBALL="cua-driver-rs-${VERSION}-${LABEL}-binary.tar.gz"+URL="https://github.com/$REPO/releases/download/$TAG/$TARBALL"++log "downloading $URL"+if ! curl -fsSL -o "$TMP_DIR/$TARBALL" "$URL"; then+ err "download failed; try CUA_DRIVER_RS_VERSION=<version> to pin a specific release"+ exit 1+fi++log "extracting"+tar -xzf "$TMP_DIR/$TARBALL" -C "$TMP_DIR"++# The bare-binary tarball contains exactly `cua-driver` at the root.+SRC="$TMP_DIR/$BINARY_NAME"+if [[ ! -f "$SRC" ]]; then+ err "expected $BINARY_NAME in tarball but didn't find it"+ ls -la "$TMP_DIR"+ exit 1+fi++# --- Install ------------------------------------------------------------++mkdir -p "$BIN_DIR"+install -m 0755 "$SRC" "$BIN_LINK"+log "installed $BIN_LINK (version $VERSION)"++# Auto-extend PATH for users whose shell doesn't already include BIN_DIR.+if [[ "$NO_MODIFY_PATH" != "1" ]] && [[ ":$PATH:" != *":$BIN_DIR:"* ]]; then+ SHELL_RC=""+ case "${SHELL:-}" in+ */zsh) SHELL_RC="$HOME/.zshrc" ;;+ */bash) SHELL_RC="$HOME/.bashrc" ;;+ esac+ if [[ -n "$SHELL_RC" ]]; then+ {+ printf '\n# Added by cua-driver-rs installer — see https://github.com/trycua/cua\n'+ printf 'export PATH="%s:$PATH"\n' "$BIN_DIR"+ } >> "$SHELL_RC"+ log "appended PATH update to $SHELL_RC — open a new shell or run \`source $SHELL_RC\`"+ else+ log "WARNING: $BIN_DIR is not on PATH; add it manually."+ fi+fi++echo ""+echo "cua-driver-rs $VERSION installed."+echo ""+echo "Try it:"+echo " $BIN_LINK list-tools"+echo " $BIN_LINK list_apps"+echo ""+echo "Docs: https://github.com/trycua/cua/tree/main/libs/cua-driver-rs"+echo ""+echo "⚠️ BETA: cua-driver-rs is a cross-platform Rust port of the Swift"+echo " cua-driver. Windows and Linux support is feature-complete; macOS"+echo " parity with the Swift binary is in progress. For production macOS"+echo " use, prefer the original install:"+echo " /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)\""diff --git a/libs/cua-driver-rs/test-apps/.gitignore b/libs/cua-driver-rs/test-apps/.gitignorenew file mode 100644index 0000000000..509bbf7522--- /dev/null+++ b/libs/cua-driver-rs/test-apps/.gitignore@@ -0,0 +1,8 @@+# Pre-built test app binaries are downloaded by run-tests-in-sandbox.ps1+# Run: .\sandbox\run-tests-in-sandbox.ps1 to auto-download before testing+*.exe+*.app+*.dmg+*.zip+*.AppImage+*.debdiff --git a/libs/cua-driver-rs/test-apps/README.md b/libs/cua-driver-rs/test-apps/README.mdnew file mode 100644index 0000000000..6b2419a5ad--- /dev/null+++ b/libs/cua-driver-rs/test-apps/README.md@@ -0,0 +1,15 @@+# test-apps++Pre-built test application binaries used by `ux_guard_test`.++## Contents++- `desktop-test-app-electron.0.1.0.exe` — Electron test app from+ [trycua/desktop-test-app-electron v0.1.0](https://github.com/trycua/desktop-test-app-electron/releases/tag/v0.1.0).+ **Not committed to git** (76 MB). Downloaded automatically by+ `sandbox\run-tests-in-sandbox.ps1` on first run.++## Updating++To update to a new version, edit the download URL in+`sandbox\run-tests-in-sandbox.ps1` and delete the old binary.diff --git a/libs/cua-driver-rs/tests/integration/driver_client.py b/libs/cua-driver-rs/tests/integration/driver_client.pynew file mode 100644index 0000000000..850b16abeb--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/driver_client.py@@ -0,0 +1,214 @@+"""Minimal MCP stdio client used by cua-driver-rs integration tests.++Speaks JSON-RPC 2.0 line-framed over stdin/stdout. Keeps request IDs monotonic+and strips notifications from the response stream. Deliberately tiny — we want+the test failures to point at cua-driver-rs, not at a heavy client dependency.+"""++from __future__ import annotations++import json+import os+import subprocess+import threading+import time+from collections import deque+from typing import Any, Optional+++class DriverClient:+ def __init__(self, binary_path: str, subcommand: str = "mcp") -> None:+ self.binary_path = binary_path+ self.subcommand = subcommand+ self.process: Optional[subprocess.Popen] = None+ self._next_id = 0+ self._lines: deque[str] = deque()+ self._reader: Optional[threading.Thread] = None+ self._stderr_reader: Optional[threading.Thread] = None++ def __enter__(self) -> "DriverClient":+ self.process = subprocess.Popen(+ [self.binary_path, self.subcommand],+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ text=True,+ bufsize=1,+ )+ self._reader = threading.Thread(target=self._pump_stdout, daemon=True)+ self._reader.start()+ # Drain stderr to prevent pipe-buffer deadlock: the driver writes tracing+ # logs to stderr; if the 64KB OS pipe buffer fills the driver blocks and+ # the test hangs indefinitely waiting for an MCP response that never comes.+ self._stderr_reader = threading.Thread(target=self._drain_stderr, daemon=True)+ self._stderr_reader.start()+ self._handshake()+ return self++ def __exit__(self, *exc: Any) -> None:+ if not self.process:+ return+ for stream in (self.process.stdin, self.process.stdout, self.process.stderr):+ try:+ if stream is not None:+ stream.close()+ except Exception:+ pass+ try:+ self.process.terminate()+ self.process.wait(timeout=3)+ except Exception:+ self.process.kill()+ self.process.wait(timeout=1)++ def _pump_stdout(self) -> None:+ assert self.process and self.process.stdout+ for line in self.process.stdout:+ line = line.strip()+ if line:+ self._lines.append(line)++ def _drain_stderr(self) -> None:+ """Discard stderr output to prevent pipe-buffer deadlock."""+ assert self.process and self.process.stderr+ try:+ for _ in self.process.stderr:+ pass+ except Exception:+ pass++ def _handshake(self) -> None:+ self._call(+ "initialize",+ {+ "protocolVersion": "2025-06-18",+ "capabilities": {},+ "clientInfo": {"name": "cua-driver-rs-integration", "version": "0.0.1"},+ },+ )+ self._notify("notifications/initialized")++ def _notify(self, method: str, params: Optional[dict] = None) -> None:+ payload = {"jsonrpc": "2.0", "method": method}+ if params is not None:+ payload["params"] = params+ self._write(payload)++ def _call(+ self, method: str, params: Optional[dict] = None, timeout: float = 20.0+ ) -> dict:+ self._next_id += 1+ request_id = self._next_id+ payload = {"jsonrpc": "2.0", "id": request_id, "method": method}+ if params is not None:+ payload["params"] = params+ self._write(payload)++ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ while self._lines:+ line = self._lines.popleft()+ try:+ msg = json.loads(line)+ except json.JSONDecodeError:+ continue+ if msg.get("id") == request_id:+ if "error" in msg:+ raise MCPCallError(msg["error"])+ return msg["result"]+ time.sleep(0.02)+ raise TimeoutError(f"no response for {method} within {timeout}s")++ def _write(self, payload: dict) -> None:+ assert self.process and self.process.stdin+ self.process.stdin.write(json.dumps(payload) + "\n")+ self.process.stdin.flush()++ def list_tools(self) -> list[dict]:+ return self._call("tools/list")["tools"]++ def call_tool(self, name: str, arguments: Optional[dict] = None) -> dict:+ return self._call(+ "tools/call",+ {"name": name, "arguments": arguments or {}},+ )+++class MCPCallError(RuntimeError):+ def __init__(self, error: dict) -> None:+ super().__init__(f"MCP error {error.get('code')}: {error.get('message')}")+ self.code = error.get("code")+ self.message = error.get("message")+++def default_binary_path() -> str:+ """Return the path to the cua-driver-rs binary.++ Checks CUA_DRIVER_BINARY env var first, then falls back to the debug build+ in the Rust workspace target directory.+ """+ return os.environ.get(+ "CUA_DRIVER_BINARY",+ os.path.join(+ os.path.dirname(os.path.abspath(__file__)),+ "..",+ "..",+ "target",+ "debug",+ "cua-driver",+ ),+ )+++def reset_calculator(settle_s: float = 0.3) -> None:+ """Force a known clean state for Calculator.++ Several tests drive Calculator as a 'freshly launched, never frontmost'+ target. `pkill -x Calculator` + short settle is the shared boilerplate.+ """+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(settle_s)+++def frontmost_bundle_id(client: "DriverClient") -> Optional[str]:+ """Return the bundle id of the currently-frontmost app, or None."""+ apps = client.call_tool("list_apps")["structuredContent"]["apps"]+ for app in apps:+ if app.get("active"):+ return app.get("bundle_id")+ return None+++def resolve_window_id(+ client: "DriverClient", pid: int, require_on_current_space: bool = True+) -> int:+ """Pick a `window_id` for `pid` via `list_windows`.++ Prefers windows that are on-screen AND on the user's current Space,+ then max z_index. Falls back to any layer-0 window for the pid when+ the preferred filter finds nothing (hidden-launched / minimized apps+ still have layer-0 windows we can snapshot).++ Raises `RuntimeError` when the pid has no layer-0 window at all.+ """+ result = client.call_tool("list_windows", {"pid": pid})+ windows = result["structuredContent"]["windows"]+ if not windows:+ raise RuntimeError(f"pid {pid} has no windows")++ if require_on_current_space:+ preferred = [+ w for w in windows+ if w.get("is_on_screen") and w.get("on_current_space") is not False+ ]+ if preferred:+ preferred.sort(key=lambda w: w.get("z_index", 0), reverse=True)+ return preferred[0]["window_id"]++ # Fallback: any layer-0 window. Pick the one with the biggest bounds.+ def _area(w: dict) -> float:+ b = w.get("bounds", {})+ return float(b.get("width", 0)) * float(b.get("height", 0))++ windows.sort(key=_area, reverse=True)+ return windows[0]["window_id"]diff --git a/libs/cua-driver-rs/tests/integration/fixtures/interactive.html b/libs/cua-driver-rs/tests/integration/fixtures/interactive.htmlnew file mode 100644index 0000000000..e67b6fee1f--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/fixtures/interactive.html@@ -0,0 +1,31 @@+<!DOCTYPE html>+<html>+<head>+<meta charset="utf-8">+<title>CUA Driver Test Page</title>+<style>+body { font-family: -apple-system, sans-serif; padding: 40px; background: #f5f5f5; }+#counter { font-size: 48px; font-weight: bold; margin: 20px 0; }+button { font-size: 24px; padding: 12px 32px; cursor: pointer; }+input { font-size: 24px; padding: 12px; width: 400px; margin-top: 20px; display: block; }+#typed { font-size: 20px; color: #333; margin-top: 10px; }+</style>+</head>+<body>+<h1>Interactive Test Page</h1>+<div id="counter">clicks: 0</div>+<button id="clicker" onclick="bump()">Click Me</button>+<input id="textbox" type="text" placeholder="Type here..." oninput="mirror()">+<div id="typed">typed: </div>+<script>+var n = 0;+function bump() {+ n++;+ document.getElementById('counter').textContent = 'clicks: ' + n;+}+function mirror() {+ document.getElementById('typed').textContent = 'typed: ' + document.getElementById('textbox').value;+}+</script>+</body>+</html>diff --git a/libs/cua-driver-rs/tests/integration/fixtures/test_cursor.png b/libs/cua-driver-rs/tests/integration/fixtures/test_cursor.pngnew file mode 100644index 0000000000..72129ae29aBinary files /dev/null and b/libs/cua-driver-rs/tests/integration/fixtures/test_cursor.png differdiff --git a/libs/cua-driver-rs/tests/integration/run_tests.sh b/libs/cua-driver-rs/tests/integration/run_tests.shnew file mode 100755index 0000000000..0a1d3c6052--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/run_tests.sh@@ -0,0 +1,87 @@+#!/usr/bin/env bash+# Run cua-driver-rs integration tests.+#+# Usage:+# ./run_tests.sh # run all tests (Rust binary)+# ./run_tests.sh test_list_windows # run a specific test module+# ./run_tests.sh test_cli test_api_parity # run multiple modules+# ./run_tests.sh --parity # run parity suite against BOTH binaries+#+# The script auto-builds cua-driver (debug) before running unless+# CUA_DRIVER_BINARY is already set in the environment.+#+# Options:+# -v / --verbose Pass -v to unittest for verbose output.+# --release Build with --release (sets CUA_DRIVER_BINARY to release binary).+# --parity Run test_api_parity.py against both Swift and Rust binaries.+# Requires CUA_SWIFT_BINARY or ~/.local/bin/cua-driver.+#+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+WORKSPACE_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"++VERBOSE=""+PROFILE="debug"+PARITY_MODE=0+MODULES=()++for arg in "$@"; do+ case "$arg" in+ -v|--verbose) VERBOSE="-v" ;;+ --release) PROFILE="release" ;;+ --parity) PARITY_MODE=1 ;;+ test_*) MODULES+=("$arg") ;;+ *) echo "Unknown argument: $arg"; exit 1 ;;+ esac+done++# Build the binary if CUA_DRIVER_BINARY is not already set.+if [ -z "${CUA_DRIVER_BINARY:-}" ]; then+ echo "==> Building cua-driver ($PROFILE)..."+ CARGO_FLAGS=""+ [ "$PROFILE" = "release" ] && CARGO_FLAGS="--release"+ (cd "$WORKSPACE_ROOT" && cargo build $CARGO_FLAGS -p cua-driver 2>&1)+ export CUA_DRIVER_BINARY="$WORKSPACE_ROOT/target/$PROFILE/cua-driver"+fi++echo "==> Using Rust binary: $CUA_DRIVER_BINARY"+if [ ! -x "$CUA_DRIVER_BINARY" ]; then+ echo "ERROR: binary not found or not executable: $CUA_DRIVER_BINARY"+ exit 1+fi++# Auto-detect Swift binary for parity mode.+if [ "$PARITY_MODE" -eq 1 ] && [ -z "${CUA_SWIFT_BINARY:-}" ]; then+ for candidate in "$HOME/.local/bin/cua-driver" "$(which cua-driver 2>/dev/null || true)"; do+ if [ -n "$candidate" ] && [ -x "$candidate" ]; then+ export CUA_SWIFT_BINARY="$candidate"+ break+ fi+ done+ if [ -z "${CUA_SWIFT_BINARY:-}" ]; then+ echo "WARNING: --parity requested but no Swift binary found."+ echo " Install cua-driver or set CUA_SWIFT_BINARY."+ echo " SwiftParityTests will be skipped."+ else+ echo "==> Using Swift binary: $CUA_SWIFT_BINARY"+ fi+fi++cd "$SCRIPT_DIR"++if [ "$PARITY_MODE" -eq 1 ]; then+ echo "==> Running parity suite against both binaries..."+ python3 -m unittest ${VERBOSE} test_api_parity+ exit $?+fi++if [ "${#MODULES[@]}" -eq 0 ]; then+ # Discover all test_*.py files.+ mapfile -t MODULES < <(find . -maxdepth 1 -name 'test_*.py' -exec basename {} .py \; | sort)+fi++echo "==> Running ${#MODULES[@]} test module(s): ${MODULES[*]}"+echo ""++python3 -m unittest ${VERBOSE} "${MODULES[@]}"diff --git a/libs/cua-driver-rs/tests/integration/test_api_parity.py b/libs/cua-driver-rs/tests/integration/test_api_parity.pynew file mode 100644index 0000000000..3fbe1c2c50--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_api_parity.py@@ -0,0 +1,1206 @@+"""Binary-level API parity tests: Swift cua-driver vs Rust cua-driver-rs.++Covers every public API surface of both binaries:+ • CLI subcommands (list-tools, describe, call, mcp-config, serve/stop/status,+ recording, dump-docs, update, doctor, diagnose, config, --version, --help)+ • Stdio MCP protocol (tools/list + every registered tool)+ • Embedded daemon mode (serve socket lifecycle)++Both binaries are tested via the *same* mixin class so any divergence shows+up as a test failure attributed to the specific binary.++Environment variables:+ CUA_SWIFT_BINARY – path to the Swift binary+ (default: ~/.local/bin/cua-driver or `which cua-driver`)+ CUA_DRIVER_BINARY – path to the Rust binary+ (default: ../../target/debug/cua-driver)++Run all parity tests against both binaries::++ python3 -m unittest test_api_parity -v++Run against Rust binary only::++ CUA_DRIVER_BINARY=../../target/release/cua-driver \\+ python3 -m unittest test_api_parity.RustParityTests -v++Run against Swift binary only::++ python3 -m unittest test_api_parity.SwiftParityTests -v+"""++from __future__ import annotations++import json+import os+import shutil+import subprocess+import sys+import tempfile+import time+import unittest++_HERE = os.path.dirname(os.path.abspath(__file__))+sys.path.insert(0, _HERE)++from driver_client import DriverClient, MCPCallError, default_binary_path # noqa: E402++# ── helpers ────────────────────────────────────────────────────────────────────++def _run(+ args: list[str],+ timeout: int = 20,+ stdin: str | None = None,+) -> subprocess.CompletedProcess:+ return subprocess.run(+ args,+ capture_output=True,+ text=True,+ timeout=timeout,+ input=stdin,+ )+++def _swift_binary() -> str | None:+ """Return Swift cua-driver binary path, or None if not found."""+ from_env = os.environ.get("CUA_SWIFT_BINARY", "")+ if from_env and os.path.isfile(from_env):+ return from_env+ candidates = [+ os.path.expanduser("~/.local/bin/cua-driver"),+ ]+ found = shutil.which("cua-driver")+ if found:+ candidates.insert(0, found)+ for p in candidates:+ if os.path.isfile(p) and os.access(p, os.X_OK):+ return p+ return None+++# ── tool presence lists ────────────────────────────────────────────────────────+#+# REQUIRED_TOOLS – tools that BOTH binaries must expose (tests fail on either+# binary when a listed tool is absent).+# RUST_ONLY_TOOLS – tools in Rust but not (yet) in Swift. Tests annotated+# with these are expected to fail on Swift.+# SWIFT_ONLY_TOOLS – tools in Swift but not (yet) registered in Rust. Tests+# annotated with these are expected to fail on Rust.+#+# Update these lists as parity gaps close.++REQUIRED_TOOLS = [+ # Core enumeration+ "get_screen_size",+ "screenshot",+ "list_apps",+ "list_windows",+ "get_window_state",+ # Interaction+ "click",+ "double_click",+ "right_click",+ "drag",+ "type_text",+ "press_key",+ "hotkey",+ "scroll",+ "set_value",+ "zoom",+ # App lifecycle+ "launch_app",+ # Permissions+ "check_permissions",+ # Cursor overlay+ "move_cursor",+ "get_cursor_position",+ "set_agent_cursor_style",+ "get_agent_cursor_state",+ "set_agent_cursor_enabled",+ "set_agent_cursor_motion",+ # Config+ "get_config",+ "set_config",+ # Recording / replay+ "set_recording",+ "get_recording_state",+ "replay_trajectory",+]++# In Rust, not in Swift yet. Tests for these will fail on the Swift binary.+RUST_ONLY_TOOLS = [+ "type_text_chars", # per-character delay typing; Swift uses type_text+ "browser_eval", # CDP JS eval; Swift uses 'page' for browser JS+ "get_accessibility_tree",# lightweight desktop AX snapshot (separate from get_window_state)+]++# In Swift, not registered in Rust yet (page.rs exists but not in register_all).+SWIFT_ONLY_TOOLS = [+ "page", # browser JS + get_text + query_dom; Rust uses browser_eval+]+++# ── parity mixin ───────────────────────────────────────────────────────────────++class _ParityMixin:+ """All API surface tests. Concrete subclasses set `cls.binary`."""++ binary: str = "" # set by setUpClass++ # ── CLI: version / help ───────────────────────────────────────────────────++ def test_version_flag_exits_zero(self) -> None:+ """--version must exit 0 (even if the binary enters MCP mode silently)."""+ r = _run([self.binary, "--version"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")++ def test_version_flag_outputs_version_number(self) -> None:+ """--version must print a semver string to stdout or stderr.++ Parity gap: Rust binary currently exits 0 with no output because+ '--version' is treated as an unknown bare flag and silently enters+ MCP mode. This test documents the requirement; it will fail until+ '--version' is handled explicitly in cli::parse_command.+ """+ r = _run([self.binary, "--version"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertRegex(+ r.stdout + r.stderr,+ r"\d+\.\d+\.\d+",+ "--version did not print a semver string (N.N.N)",+ )++ def test_help_flag_exits_zero(self) -> None:+ r = _run([self.binary, "--help"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertIn("cua-driver", r.stdout + r.stderr)++ # ── CLI: list-tools ───────────────────────────────────────────────────────++ def test_list_tools_exits_zero(self) -> None:+ r = _run([self.binary, "list-tools"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")++ def test_list_tools_contains_required_tool_names(self) -> None:+ r = _run([self.binary, "list-tools"])+ self.assertEqual(r.returncode, 0)+ for name in [+ "click", "screenshot", "list_windows", "press_key",+ "get_window_state", "type_text", "hotkey", "scroll",+ ]:+ self.assertIn(name, r.stdout, f"list-tools missing: {name!r}")++ def test_list_tools_every_required_tool_present(self) -> None:+ r = _run([self.binary, "list-tools"])+ self.assertEqual(r.returncode, 0)+ for name in REQUIRED_TOOLS:+ self.assertIn(name, r.stdout, f"list-tools missing: {name!r}")++ # ── CLI: describe ─────────────────────────────────────────────────────────++ def test_describe_click_has_pid_and_schema(self) -> None:+ r = _run([self.binary, "describe", "click"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertIn("click", r.stdout)+ self.assertIn("pid", r.stdout)+ self.assertIn("input_schema", r.stdout)++ def test_describe_screenshot_has_schema(self) -> None:+ r = _run([self.binary, "describe", "screenshot"])+ self.assertEqual(r.returncode, 0)+ self.assertIn("screenshot", r.stdout)+ self.assertIn("input_schema", r.stdout)++ def test_describe_each_required_tool_exits_zero(self) -> None:+ for name in REQUIRED_TOOLS:+ with self.subTest(tool=name):+ r = _run([self.binary, "describe", name])+ self.assertEqual(+ r.returncode, 0,+ f"describe {name!r} exited {r.returncode}: {r.stderr}",+ )+ self.assertIn(name, r.stdout)+ self.assertIn("input_schema", r.stdout)++ def test_describe_unknown_tool_exits_64(self) -> None:+ r = _run([self.binary, "describe", "no_such_tool_xyzzy"])+ self.assertEqual(r.returncode, 64)+ self.assertIn("no_such_tool_xyzzy", r.stderr)++ # ── CLI: call (stateless tools) ───────────────────────────────────────────++ def test_call_get_screen_size_exits_zero(self) -> None:+ r = _run([self.binary, "call", "get_screen_size"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")++ def test_call_get_screen_size_returns_json(self) -> None:+ """call get_screen_size must emit JSON with width and height.++ Parity gap: Swift binary outputs human-readable text+ (e.g. "✅ Main display: 1920x1080 points @ 1.0x") instead of JSON.+ This test documents the requirement; it fails on Swift.+ """+ r = _run([self.binary, "call", "get_screen_size"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ try:+ data = json.loads(r.stdout)+ except json.JSONDecodeError:+ self.fail(+ f"call get_screen_size did not output JSON.\n"+ f"stdout: {r.stdout!r}\n"+ f"(Swift binary outputs human-readable text instead of JSON — parity gap)"+ )+ self.assertIn("width", data)+ self.assertIn("height", data)+ self.assertGreater(data["width"], 0)+ self.assertGreater(data["height"], 0)++ def test_call_screenshot_returns_b64_png(self) -> None:+ """call screenshot (no args) must return JSON with screenshot_png_b64.++ Parity gap: Swift binary requires `window_id` for screenshot and+ returns a non-JSON error when called without args. The Rust binary+ defaults to a full-display screenshot without requiring window_id.+ This test fails on Swift on both counts; it documents the requirement.+ """+ r = _run([self.binary, "call", "screenshot"], timeout=30)+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}\nstdout: {r.stdout!r}")+ try:+ data = json.loads(r.stdout)+ except json.JSONDecodeError:+ self.fail(+ f"call screenshot did not output JSON.\n"+ f"stdout: {r.stdout!r}\n"+ f"(Swift requires window_id; Rust defaults to full display)"+ )+ self.assertIn("screenshot_png_b64", data)+ self.assertGreater(len(data["screenshot_png_b64"]), 500)++ def test_call_list_apps_returns_app_list(self) -> None:+ r = _run([self.binary, "call", "list_apps"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("apps", data)+ self.assertIsInstance(data["apps"], list)+ self.assertGreater(len(data["apps"]), 0)++ def test_call_list_apps_entries_have_required_fields(self) -> None:+ r = _run([self.binary, "call", "list_apps"])+ self.assertEqual(r.returncode, 0)+ data = json.loads(r.stdout)+ for app in data["apps"][:5]: # spot-check first 5+ with self.subTest(app=app.get("name") or app.get("bundle_id")):+ self.assertIn("pid", app)+ self.assertIn("name", app)++ def test_call_list_windows_returns_window_list(self) -> None:+ r = _run([self.binary, "call", "list_windows"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("windows", data)+ self.assertIsInstance(data["windows"], list)++ def test_call_list_windows_entries_have_required_fields(self) -> None:+ r = _run([self.binary, "call", "list_windows"])+ self.assertEqual(r.returncode, 0)+ data = json.loads(r.stdout)+ for w in data["windows"][:5]:+ with self.subTest(window_id=w.get("window_id")):+ self.assertIn("window_id", w)+ self.assertIn("pid", w)+ self.assertIn("bounds", w)+ b = w["bounds"]+ for field in ("x", "y", "width", "height"):+ self.assertIn(field, b)++ def test_call_check_permissions_exits_zero(self) -> None:+ r = _run([self.binary, "call", "check_permissions"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")++ def test_call_check_permissions_returns_accessibility_and_screen_recording(+ self,+ ) -> None:+ """call check_permissions must emit JSON with accessibility/screen_recording.++ Parity gap: Swift binary outputs human-readable text with emoji+ (✅/❌) instead of JSON. This test documents the JSON requirement+ and will fail on Swift until its CLI output is standardised.+ """+ r = _run([self.binary, "call", "check_permissions"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ try:+ data = json.loads(r.stdout)+ except json.JSONDecodeError:+ self.fail(+ f"call check_permissions did not output JSON.\n"+ f"stdout: {r.stdout!r}"+ )+ self.assertIn("accessibility", data)+ self.assertIn("screen_recording", data)++ def test_call_get_cursor_position_returns_xy(self) -> None:+ r = _run([self.binary, "call", "get_cursor_position"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("x", data)+ self.assertIn("y", data)++ def test_call_get_config_returns_config_object(self) -> None:+ r = _run([self.binary, "call", "get_config"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ # Config must have at least one of the known top-level keys.+ known_keys = {+ "schema_version", "capture_mode", "max_image_dimension",+ "agent_cursor", "telemetry_enabled",+ }+ self.assertTrue(+ known_keys.intersection(data.keys()),+ f"get_config returned no known keys: {list(data.keys())}",+ )++ def test_call_get_agent_cursor_state_returns_state(self) -> None:+ r = _run([self.binary, "call", "get_agent_cursor_state"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("enabled", data)++ def test_call_get_recording_state_returns_state(self) -> None:+ r = _run([self.binary, "call", "get_recording_state"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("recording", data)++ # ── CLI: call with a pid-bearing tool (launch_app + get_window_state) ─────++ def test_call_launch_app_calculator_returns_pid(self) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)+ r = _run(+ [self.binary, "call", "launch_app", '{"bundle_id": "com.apple.calculator"}'],+ timeout=20,+ )+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("pid", data)+ self.assertGreater(data["pid"], 0)++ def test_call_get_window_state_for_calculator(self) -> None:+ """launch Calculator, get its window, then fetch get_window_state."""+ # Launch+ rl = _run(+ [self.binary, "call", "launch_app", '{"bundle_id": "com.apple.calculator"}'],+ timeout=20,+ )+ self.assertEqual(rl.returncode, 0, f"launch stderr: {rl.stderr}")+ pid = json.loads(rl.stdout)["pid"]+ time.sleep(1.0)++ # List windows to get a window_id.+ rw = _run(+ [self.binary, "call", "list_windows", f'{{"pid": {pid}}}'],+ )+ self.assertEqual(rw.returncode, 0, f"list_windows stderr: {rw.stderr}")+ windows = json.loads(rw.stdout)["windows"]+ self.assertTrue(windows, "Calculator has no windows")+ window_id = windows[0]["window_id"]++ # get_window_state+ rs = _run(+ [self.binary, "call", "get_window_state",+ f'{{"pid": {pid}, "window_id": {window_id}}}'],+ timeout=30,+ )+ self.assertEqual(rs.returncode, 0, f"get_window_state stderr: {rs.stderr}")+ data = json.loads(rs.stdout)+ self.assertIn("tree_markdown", data)+ self.assertIn("AX", data["tree_markdown"])++ # ── CLI: call — error / missing args paths ────────────────────────────────++ def test_call_unknown_tool_exits_64(self) -> None:+ r = _run([self.binary, "call", "no_such_tool_xyzzy"])+ self.assertEqual(r.returncode, 64)++ def test_call_press_key_missing_pid_exits_1(self) -> None:+ r = _run([self.binary, "call", "press_key", '{"key": "a"}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}")++ def test_call_click_missing_required_args_exits_1(self) -> None:+ r = _run([self.binary, "call", "click", '{}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}")++ def test_call_type_text_missing_pid_exits_1(self) -> None:+ r = _run([self.binary, "call", "type_text", '{"text": "hello"}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}")++ def test_call_hotkey_missing_pid_exits_1(self) -> None:+ r = _run([self.binary, "call", "hotkey", '{"keys": ["cmd", "a"]}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}")++ def test_call_scroll_missing_required_args_exits_1(self) -> None:+ r = _run([self.binary, "call", "scroll", '{}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}")++ def test_call_drag_missing_required_args_exits_1(self) -> None:+ r = _run([self.binary, "call", "drag", '{}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}")++ def test_call_get_window_state_missing_pid_exits_1(self) -> None:+ r = _run([self.binary, "call", "get_window_state", '{"window_id": 1}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}")++ def test_call_launch_app_no_args_exits_1(self) -> None:+ r = _run([self.binary, "call", "launch_app", '{}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}")++ # ── CLI: implicit call (tool name as first positional) ────────────────────++ def test_implicit_call_get_screen_size(self) -> None:+ r = _run([self.binary, "get_screen_size"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("width", data)+ self.assertIn("height", data)++ def test_implicit_call_check_permissions(self) -> None:+ r = _run([self.binary, "check_permissions"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("accessibility", data)++ def test_implicit_call_list_apps(self) -> None:+ r = _run([self.binary, "list_apps"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("apps", data)++ def test_implicit_call_screenshot_exits_zero(self) -> None:+ """Implicit-call screenshot must exit 0.++ Note: Swift binary requires window_id so exits non-zero with an+ error message — this is a parity gap in Swift.+ """+ r = _run([self.binary, "screenshot"], timeout=30)+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}\nstdout: {r.stdout!r}")++ def test_implicit_unknown_exits_64(self) -> None:+ r = _run([self.binary, "no_such_tool_xyzzy"])+ self.assertEqual(r.returncode, 64)++ # ── CLI: mcp-config ───────────────────────────────────────────────────────++ def test_mcp_config_default_outputs_json(self) -> None:+ r = _run([self.binary, "mcp-config"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ out = r.stdout.strip()+ # Should either be a JSON object or a shell snippet containing "cua-driver"+ self.assertIn("cua-driver", out)++ def test_mcp_config_client_claude_outputs_add_command(self) -> None:+ r = _run([self.binary, "mcp-config", "--client", "claude"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertIn("cua-driver", r.stdout)++ def test_mcp_config_client_cursor_exits_zero(self) -> None:+ r = _run([self.binary, "mcp-config", "--client", "cursor"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")++ # ── CLI: dump-docs ────────────────────────────────────────────────────────++ def test_dump_docs_exits_zero(self) -> None:+ r = _run([self.binary, "dump-docs"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertTrue(len(r.stdout) > 200, "dump-docs output too short")++ def test_dump_docs_contains_tool_names(self) -> None:+ r = _run([self.binary, "dump-docs"])+ self.assertEqual(r.returncode, 0)+ for name in ["click", "screenshot", "get_window_state"]:+ self.assertIn(name, r.stdout)++ def test_dump_docs_pretty_is_valid_json(self) -> None:+ r = _run([self.binary, "dump-docs", "--pretty"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ # --pretty outputs JSON+ try:+ data = json.loads(r.stdout)+ except json.JSONDecodeError:+ self.fail(f"dump-docs --pretty is not valid JSON:\n{r.stdout[:500]}")+ self.assertIsInstance(data, (list, dict))++ # ── CLI: diagnose ─────────────────────────────────────────────────────────++ def test_diagnose_exits_zero(self) -> None:+ r = _run([self.binary, "diagnose"], timeout=30)+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertTrue(len(r.stdout) > 50)++ def test_diagnose_mentions_accessibility(self) -> None:+ r = _run([self.binary, "diagnose"], timeout=30)+ self.assertEqual(r.returncode, 0)+ out = (r.stdout + r.stderr).lower()+ self.assertIn("access", out)++ # ── CLI: doctor ───────────────────────────────────────────────────────────++ def test_doctor_exits_zero_or_one(self) -> None:+ """doctor returns 0 if all checks pass, 1 if any check fails."""+ r = _run([self.binary, "doctor"], timeout=20)+ self.assertIn(r.returncode, (0, 1), f"unexpected exit: {r.returncode}")++ def test_doctor_produces_output(self) -> None:+ r = _run([self.binary, "doctor"], timeout=20)+ self.assertIn(r.returncode, (0, 1))+ out = r.stdout + r.stderr+ self.assertGreater(len(out.strip()), 0)++ # ── CLI: config ───────────────────────────────────────────────────────────++ def test_config_show_exits_zero(self) -> None:+ r = _run([self.binary, "config", "show"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertGreater(len(r.stdout.strip()), 0)++ def test_config_get_max_image_dimension(self) -> None:+ r = _run([self.binary, "config", "get", "max_image_dimension"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ # Value should look like an integer+ out = r.stdout.strip()+ self.assertRegex(out, r"\d+", f"expected integer, got: {out!r}")++ def test_config_get_agent_cursor_enabled(self) -> None:+ r = _run([self.binary, "config", "get", "agent_cursor.enabled"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ val = r.stdout.strip().lower()+ self.assertIn(val, ("true", "false"), f"unexpected value: {val!r}")++ def test_config_set_and_restore_max_image_dimension(self) -> None:+ # Read original value.+ r0 = _run([self.binary, "config", "get", "max_image_dimension"])+ self.assertEqual(r0.returncode, 0)+ original = r0.stdout.strip()++ # Set a new value.+ r1 = _run([self.binary, "config", "set", "max_image_dimension", "1024"])+ self.assertEqual(r1.returncode, 0, f"set stderr: {r1.stderr}")++ # Confirm it was applied.+ r2 = _run([self.binary, "config", "get", "max_image_dimension"])+ self.assertEqual(r2.returncode, 0)+ self.assertEqual(r2.stdout.strip(), "1024")++ # Restore original value.+ _run([self.binary, "config", "set", "max_image_dimension", original])++ def test_config_reset_exits_zero(self) -> None:+ r = _run([self.binary, "config", "reset"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")++ def test_config_get_unknown_key_exits_nonzero(self) -> None:+ r = _run([self.binary, "config", "get", "no_such_key_xyzzy"])+ self.assertNotEqual(r.returncode, 0)++ # ── CLI: update ───────────────────────────────────────────────────────────++ def test_update_dry_run_exits_zero(self) -> None:+ """update without --apply should exit 0 even if already up-to-date."""+ r = _run([self.binary, "update"], timeout=30)+ self.assertIn(r.returncode, (0, 1), f"unexpected exit: {r.returncode}")+ out = r.stdout + r.stderr+ # Should mention version or "up to date" or similar+ self.assertGreater(len(out.strip()), 0)++ # ── CLI: serve / stop / status daemon lifecycle ───────────────────────────++ def _tmp_socket(self) -> str:+ return tempfile.mktemp(suffix=".sock", prefix="cua-parity-test-")++ def test_status_exits_1_when_no_daemon(self) -> None:+ sock = self._tmp_socket()+ r = _run([self.binary, "status", "--socket", sock])+ self.assertEqual(r.returncode, 1)+ out = (r.stdout + r.stderr).lower()+ self.assertIn("not running", out)++ def test_stop_exits_1_when_no_daemon(self) -> None:+ sock = self._tmp_socket()+ r = _run([self.binary, "stop", "--socket", sock])+ self.assertEqual(r.returncode, 1)++ def test_serve_status_stop_lifecycle(self) -> None:+ sock = self._tmp_socket()+ proc = subprocess.Popen(+ [self.binary, "serve", "--socket", sock],+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ )+ try:+ # Wait for daemon to bind (up to 5 s).+ deadline = time.monotonic() + 5.0+ running = False+ while time.monotonic() < deadline:+ r = _run([self.binary, "status", "--socket", sock])+ if r.returncode == 0:+ running = True+ break+ time.sleep(0.1)+ self.assertTrue(running, "daemon did not become ready in 5 s")++ # status reports "running" and mentions the socket path.+ r = _run([self.binary, "status", "--socket", sock])+ self.assertEqual(r.returncode, 0, f"status stderr: {r.stderr}")+ out = (r.stdout + r.stderr).lower()+ self.assertIn("running", out)++ # stop the daemon.+ r = _run([self.binary, "stop", "--socket", sock])+ self.assertEqual(r.returncode, 0, f"stop stderr: {r.stderr}")++ # status should report not running.+ time.sleep(0.3)+ r = _run([self.binary, "status", "--socket", sock])+ self.assertEqual(r.returncode, 1)+ finally:+ proc.terminate()+ proc.wait(timeout=3)++ def test_serve_second_instance_exits_1(self) -> None:+ sock = self._tmp_socket()+ proc = subprocess.Popen(+ [self.binary, "serve", "--socket", sock],+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ )+ try:+ # Wait for first daemon.+ deadline = time.monotonic() + 5.0+ while time.monotonic() < deadline:+ r = _run([self.binary, "status", "--socket", sock])+ if r.returncode == 0:+ break+ time.sleep(0.1)++ # Second serve should fail.+ r = _run([self.binary, "serve", "--socket", sock])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}\nstderr: {r.stderr}")+ out = (r.stdout + r.stderr).lower()+ self.assertIn("already running", out)+ finally:+ _run([self.binary, "stop", "--socket", sock])+ proc.wait(timeout=3)++ # ── stdio MCP: tools/list ─────────────────────────────────────────────────++ def _mcp(self) -> DriverClient:+ return DriverClient(self.binary)++ def test_mcp_tools_list_returns_all_required_tools(self) -> None:+ with self._mcp() as c:+ tools = c.list_tools()+ names = {t["name"] for t in tools}+ missing = [n for n in REQUIRED_TOOLS if n not in names]+ self.assertFalse(+ missing,+ f"tools/list missing: {missing}\n got: {sorted(names)}",+ )++ def test_mcp_tools_list_each_has_name_description_schema(self) -> None:+ with self._mcp() as c:+ tools = c.list_tools()+ for t in tools:+ with self.subTest(tool=t.get("name")):+ self.assertIn("name", t)+ self.assertIn("description", t)+ self.assertIn("inputSchema", t)+ schema = t["inputSchema"]+ self.assertIn("type", schema)+ self.assertEqual(schema["type"], "object")++ # ── stdio MCP: stateless read tools ──────────────────────────────────────++ def test_mcp_get_screen_size(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("get_screen_size")+ sc = result.get("structuredContent", result)+ self.assertIn("width", sc)+ self.assertIn("height", sc)+ self.assertGreater(sc["width"], 0)+ self.assertGreater(sc["height"], 0)++ def test_mcp_screenshot_has_png_image(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("screenshot")+ # Should have image content.+ images = [i for i in result.get("content", []) if i.get("type") == "image"]+ sc = result.get("structuredContent", {})+ # At least one of: image in content OR screenshot_png_b64 in structuredContent.+ has_image = bool(images) or bool(sc.get("screenshot_png_b64"))+ self.assertTrue(has_image, f"screenshot returned no image: {result}")++ def test_mcp_screenshot_with_quality(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("screenshot", {"quality": 50})+ images = [i for i in result.get("content", []) if i.get("type") == "image"]+ sc = result.get("structuredContent", {})+ has_image = bool(images) or bool(sc.get("screenshot_png_b64"))+ self.assertTrue(has_image, "screenshot with quality returned no image")++ def test_mcp_list_apps(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("list_apps")+ sc = result["structuredContent"]+ self.assertIn("apps", sc)+ apps = sc["apps"]+ self.assertIsInstance(apps, list)+ self.assertGreater(len(apps), 0)+ # Every app entry must have pid + name.+ for app in apps[:5]:+ self.assertIn("pid", app)+ self.assertIn("name", app)++ def test_mcp_list_apps_has_active_flag(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("list_apps")+ apps = result["structuredContent"]["apps"]+ # At least one app should have an active field.+ has_active = any("active" in app for app in apps)+ self.assertTrue(has_active, "no app has 'active' field")++ def test_mcp_list_windows_default(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("list_windows")+ sc = result["structuredContent"]+ self.assertIn("windows", sc)+ windows = sc["windows"]+ self.assertIsInstance(windows, list)++ def test_mcp_list_windows_fields(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("list_windows")+ windows = result["structuredContent"]["windows"]+ for w in windows[:5]:+ with self.subTest(window_id=w.get("window_id")):+ self.assertIn("window_id", w)+ self.assertIn("pid", w)+ self.assertIn("bounds", w)+ for f in ("x", "y", "width", "height"):+ self.assertIn(f, w["bounds"])++ def test_mcp_list_windows_pid_filter(self) -> None:+ """list_windows with pid filter should only return windows for that pid."""+ with self._mcp() as c:+ # Get any pid that has windows.+ all_wins = c.call_tool("list_windows")["structuredContent"]["windows"]+ if not all_wins:+ self.skipTest("no windows available")+ target_pid = all_wins[0]["pid"]+ filtered = c.call_tool("list_windows", {"pid": target_pid})+ windows = filtered["structuredContent"]["windows"]+ for w in windows:+ self.assertEqual(w["pid"], target_pid)++ def test_mcp_list_windows_on_screen_only(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("list_windows", {"on_screen_only": True})+ windows = result["structuredContent"]["windows"]+ for w in windows:+ self.assertTrue(+ w.get("is_on_screen", True),+ f"on_screen_only=true returned off-screen window: {w}",+ )++ def test_mcp_check_permissions(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("check_permissions")+ sc = result.get("structuredContent", result)+ self.assertIn("accessibility", sc)+ self.assertIn("screen_recording", sc)+ self.assertIsInstance(sc["accessibility"], bool)+ self.assertIsInstance(sc["screen_recording"], bool)++ def test_mcp_get_cursor_position(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("get_cursor_position")+ sc = result.get("structuredContent", result)+ self.assertIn("x", sc)+ self.assertIn("y", sc)++ def test_mcp_get_agent_cursor_state(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("get_agent_cursor_state")+ sc = result.get("structuredContent", result)+ self.assertIn("enabled", sc)++ def test_mcp_set_agent_cursor_enabled_toggle(self) -> None:+ with self._mcp() as c:+ # Read current.+ state = c.call_tool("get_agent_cursor_state")+ enabled = state.get("structuredContent", state).get("enabled", True)+ # Toggle off.+ c.call_tool("set_agent_cursor_enabled", {"enabled": False})+ state2 = c.call_tool("get_agent_cursor_state")+ self.assertFalse(state2.get("structuredContent", state2).get("enabled", True))+ # Restore.+ c.call_tool("set_agent_cursor_enabled", {"enabled": enabled})++ def test_mcp_set_agent_cursor_style_default(self) -> None:+ with self._mcp() as c:+ # Calling with no args or default args should succeed.+ c.call_tool("set_agent_cursor_style", {})++ def test_mcp_get_config_roundtrip(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("get_config")+ sc = result.get("structuredContent", result)+ known = {"schema_version", "capture_mode", "max_image_dimension",+ "agent_cursor", "telemetry_enabled"}+ self.assertTrue(known.intersection(sc.keys()),+ f"get_config has no known keys: {list(sc.keys())}")++ def test_mcp_set_config_max_image_dimension(self) -> None:+ with self._mcp() as c:+ # Read original.+ orig = c.call_tool("get_config")+ orig_dim = orig.get("structuredContent", orig).get("max_image_dimension", 1920)+ # Set new value.+ c.call_tool("set_config", {"max_image_dimension": 800})+ # Confirm.+ updated = c.call_tool("get_config")+ new_dim = updated.get("structuredContent", updated).get("max_image_dimension")+ self.assertEqual(new_dim, 800)+ # Restore.+ c.call_tool("set_config", {"max_image_dimension": orig_dim})++ def test_mcp_get_recording_state(self) -> None:+ with self._mcp() as c:+ result = c.call_tool("get_recording_state")+ sc = result.get("structuredContent", result)+ self.assertIn("recording", sc)+ self.assertIsInstance(sc["recording"], bool)++ def test_mcp_set_recording_enable_disable(self) -> None:+ import tempfile+ with tempfile.TemporaryDirectory() as tmpdir:+ with self._mcp() as c:+ # Enable recording — output_dir is required.+ c.call_tool("set_recording", {"enabled": True, "output_dir": tmpdir})+ state = c.call_tool("get_recording_state")+ sc = state.get("structuredContent", state)+ # Accept either "recording" (Swift) or "enabled" (Rust) key.+ is_recording = sc.get("recording", sc.get("enabled", False))+ self.assertTrue(is_recording, f"recording should be enabled: {sc}")+ # Disable.+ c.call_tool("set_recording", {"enabled": False})+ state2 = c.call_tool("get_recording_state")+ sc2 = state2.get("structuredContent", state2)+ is_recording2 = sc2.get("recording", sc2.get("enabled", True))+ self.assertFalse(is_recording2, f"recording should be disabled: {sc2}")++ # ── stdio MCP: tool error contracts ──────────────────────────────────────++ def _assert_tool_raises_mcp_error(+ self, tool_name: str, args: dict, timeout: float = 10.0+ ) -> None:+ """Assert that calling `tool_name` with `args` signals an error.++ Accepts either a JSON-RPC MCPCallError (Swift behaviour) or a result+ with ``isError=true`` in the tool-content list (Rust behaviour).+ """+ with self._mcp() as c:+ try:+ result = c.call_tool(tool_name, args)+ # Rust returns isError:true in the content payload+ self.assertTrue(+ result.get("isError"),+ f"{tool_name} did not raise an error (isError missing or false)",+ )+ except MCPCallError:+ pass # Swift-style JSON-RPC error — acceptable++ def test_mcp_click_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("click", {"window_id": 1})++ def test_mcp_double_click_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("double_click", {"window_id": 1})++ def test_mcp_right_click_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("right_click", {"window_id": 1})++ def test_mcp_press_key_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("press_key", {"key": "a"})++ def test_mcp_hotkey_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("hotkey", {"keys": ["cmd", "a"]})++ def test_mcp_type_text_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("type_text", {"text": "hello"})++ def test_mcp_type_text_chars_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("type_text_chars", {"text": "hello"})++ def test_mcp_scroll_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("scroll", {"x": 100, "y": 100})++ def test_mcp_drag_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("drag", {})++ def test_mcp_get_window_state_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("get_window_state", {"window_id": 1})++ def test_mcp_launch_app_missing_args_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error("launch_app", {})++ def test_mcp_set_value_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error(+ "set_value", {"element_index": 0, "value": "x"}+ )++ def test_mcp_replay_trajectory_bad_dir_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error(+ "replay_trajectory", {"dir": "/tmp/no_such_trajectory_xyzzy"}+ )++ # ── stdio MCP: get_window_state live ─────────────────────────────────────++ def test_mcp_get_window_state_calculator(self) -> None:+ """Launch Calculator, capture a window state, verify tree_markdown."""+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)++ with self._mcp() as c:+ # Launch+ launch = c.call_tool("launch_app", {"bundle_id": "com.apple.calculator"})+ pid = launch.get("structuredContent", launch).get("pid", 0)+ self.assertGreater(pid, 0, f"launch_app returned bad pid: {launch}")+ time.sleep(1.0)++ # Find window+ wins = c.call_tool("list_windows", {"pid": pid})["structuredContent"]["windows"]+ self.assertTrue(wins, "Calculator has no windows")+ window_id = wins[0]["window_id"]++ # get_window_state+ state = c.call_tool("get_window_state", {"pid": pid, "window_id": window_id})++ sc = state.get("structuredContent", state)+ tree = sc.get("tree_markdown", "")+ self.assertIn("AX", tree, f"tree_markdown has no AX nodes:\n{tree}")+ # Screenshot should be present.+ images = [i for i in state.get("content", []) if i.get("type") == "image"]+ self.assertTrue(images, "get_window_state returned no screenshot")++ def test_mcp_get_window_state_has_required_sc_keys(self) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)+ with self._mcp() as c:+ launch = c.call_tool("launch_app", {"bundle_id": "com.apple.calculator"})+ pid = launch.get("structuredContent", launch).get("pid", 0)+ time.sleep(1.0)+ wins = c.call_tool("list_windows", {"pid": pid})["structuredContent"]["windows"]+ if not wins:+ self.skipTest("Calculator has no windows")+ state = c.call_tool("get_window_state", {+ "pid": pid, "window_id": wins[0]["window_id"]+ })+ sc = state.get("structuredContent", state)+ for key in ("tree_markdown", "screenshot_width", "screenshot_height"):+ self.assertIn(key, sc, f"get_window_state structuredContent missing: {key!r}")++ # ── stdio MCP: launch_app ─────────────────────────────────────────────────++ def test_mcp_launch_app_by_bundle_id(self) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)+ with self._mcp() as c:+ result = c.call_tool("launch_app", {"bundle_id": "com.apple.calculator"})+ sc = result.get("structuredContent", result)+ self.assertIn("pid", sc)+ self.assertGreater(sc["pid"], 0)++ def test_mcp_launch_app_by_name(self) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)+ with self._mcp() as c:+ result = c.call_tool("launch_app", {"name": "Calculator"})+ sc = result.get("structuredContent", result)+ self.assertIn("pid", sc)+ self.assertGreater(sc["pid"], 0)++ def test_mcp_launch_app_unknown_bundle_id_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error(+ "launch_app", {"bundle_id": "com.example.no_such_app_xyzzy"}+ )++ # ── stdio MCP: set_agent_cursor_motion ────────────────────────────────────++ def test_mcp_set_agent_cursor_motion_spring(self) -> None:+ with self._mcp() as c:+ c.call_tool("set_agent_cursor_motion", {"spring": True})++ def test_mcp_set_agent_cursor_motion_arc(self) -> None:+ with self._mcp() as c:+ c.call_tool("set_agent_cursor_motion", {"arc_size": 0.5})++ # ── stdio MCP: zoom ───────────────────────────────────────────────────────++ def test_mcp_zoom_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error(+ "zoom", {"x1": 0, "y1": 0, "x2": 100, "y2": 100}+ )++ # ── stdio MCP: page ───────────────────────────────────────────────────────++ def test_mcp_page_is_registered(self) -> None:+ """'page' tool must appear in tools/list.++ Parity gap: Rust binary has page.rs implemented but the tool is not+ yet registered in platform_macos::tools::register_all. This test will+ fail on the Rust binary until page::PageTool is added to register_all.+ """+ with self._mcp() as c:+ names = {t["name"] for t in c.list_tools()}+ self.assertIn("page", names, "'page' tool is not registered")++ def test_mcp_page_missing_pid_raises_error(self) -> None:+ self._assert_tool_raises_mcp_error(+ "page", {"action": "get_text"}+ )++ # ── stdio MCP: Rust-only tools (missing in Swift) ────────────────────────++ def test_mcp_rust_only_tools_present(self) -> None:+ """Rust-only tools (type_text_chars, browser_eval, get_accessibility_tree).++ These are in the Rust binary but not yet in Swift. Both binaries+ should eventually have them. This test fails on Swift until ported.+ """+ with self._mcp() as c:+ names = {t["name"] for t in c.list_tools()}+ missing = [n for n in RUST_ONLY_TOOLS if n not in names]+ self.assertFalse(+ missing,+ f"Rust-only tools missing: {missing} (not yet in this binary)",+ )++ def test_mcp_type_text_chars_missing_pid_raises_error(self) -> None:+ """type_text_chars (Rust-only) must raise MCP error with missing pid.++ Fails on Swift binary — tool not yet ported.+ """+ self._assert_tool_raises_mcp_error("type_text_chars", {"text": "hello"})++ def test_mcp_get_accessibility_tree_exits_without_pid(self) -> None:+ """get_accessibility_tree (Rust-only) returns desktop snapshot without pid.++ Fails on Swift binary — tool not yet ported.+ """+ with self._mcp() as c:+ result = c.call_tool("get_accessibility_tree")+ sc = result.get("structuredContent", result)+ # Should return some content — desktop always has windows.+ self.assertIsInstance(sc, dict)++ # ── stdio MCP: Swift-only tools (missing/unregistered in Rust) ───────────++ def test_mcp_swift_only_tools_present(self) -> None:+ """Swift-only tools (page).++ In Swift, not yet registered in Rust (page.rs exists but not in+ register_all). Both binaries should eventually have them.+ This test fails on Rust until page::PageTool is added to register_all.+ """+ with self._mcp() as c:+ names = {t["name"] for t in c.list_tools()}+ missing = [n for n in SWIFT_ONLY_TOOLS if n not in names]+ self.assertFalse(+ missing,+ f"Swift-only tools missing: {missing} (not yet registered in this binary)",+ )++ # ── stdio MCP: browser_eval ───────────────────────────────────────────────++ def test_mcp_browser_eval_is_registered(self) -> None:+ """'browser_eval' tool must appear in tools/list.++ Parity gap: Swift binary does not have 'browser_eval' — it uses 'page'+ for browser JS. This test documents the requirement that both binaries+ expose browser_eval. Fails on Swift until it is ported.+ """+ with self._mcp() as c:+ names = {t["name"] for t in c.list_tools()}+ self.assertIn("browser_eval", names, "'browser_eval' tool is not registered")++ def test_mcp_browser_eval_missing_pid_raises_error(self) -> None:+ """browser_eval with no pid should return an MCP error."""+ self._assert_tool_raises_mcp_error(+ "browser_eval", {"expression": "1+1"}+ )++ # ── stdio MCP: protocol-level contracts ──────────────────────────────────++ def test_mcp_tools_call_unknown_tool_returns_error(self) -> None:+ """Calling an unknown tool must signal an error.++ Accepts MCPCallError (Swift / JSON-RPC error) or isError=true (Rust).+ """+ with self._mcp() as c:+ try:+ result = c.call_tool("no_such_tool_xyzzy")+ self.assertTrue(+ result.get("isError"),+ "unknown tool should set isError=true",+ )+ except MCPCallError:+ pass # JSON-RPC level error — acceptable++ def test_mcp_handles_rapid_sequential_calls(self) -> None:+ """Send 5 sequential get_screen_size calls — all must succeed."""+ with self._mcp() as c:+ for i in range(5):+ result = c.call_tool("get_screen_size")+ sc = result.get("structuredContent", result)+ self.assertIn("width", sc, f"call {i} failed: {result}")++ def test_mcp_protocol_version_echo(self) -> None:+ """The initialize response should echo an MCP protocol version."""+ # The DriverClient handshake already calls initialize; if it doesn't+ # raise an exception, the server replied correctly.+ with self._mcp() as c:+ tools = c.list_tools()+ self.assertGreater(len(tools), 0)+++# ── concrete test classes (one per binary) ────────────────────────────────────++class SwiftParityTests(_ParityMixin, unittest.TestCase):+ """Run the full parity suite against the Swift cua-driver binary."""++ @classmethod+ def setUpClass(cls) -> None:+ path = _swift_binary()+ if not path:+ raise unittest.SkipTest(+ "Swift cua-driver not found. "+ "Set CUA_SWIFT_BINARY or install cua-driver to ~/.local/bin/."+ )+ cls.binary = path+ print(f"\n[SwiftParityTests] binary: {cls.binary}", flush=True)+++class RustParityTests(_ParityMixin, unittest.TestCase):+ """Run the full parity suite against the Rust cua-driver-rs binary."""++ @classmethod+ def setUpClass(cls) -> None:+ cls.binary = default_binary_path()+ if not os.path.isfile(cls.binary):+ raise unittest.SkipTest(+ f"Rust binary not found at {cls.binary}. "+ "Run `cargo build -p cua-driver` or set CUA_DRIVER_BINARY."+ )+ print(f"\n[RustParityTests] binary: {cls.binary}", flush=True)+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/test_background_focus.py b/libs/cua-driver-rs/tests/integration/test_background_focus.pynew file mode 100644index 0000000000..b86e1fbe9a--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_background_focus.py@@ -0,0 +1,568 @@+"""Integration test: interact with backgrounded apps without stealing focus.++Setup:+ 1. Open Safari to a local HTML page with a button and text input.+ 2. Launch FocusMonitorApp (counts NSApplication.didResignActiveNotification).+ 3. FocusMonitorApp is activated last so it owns focus.++Tests send clicks and keystrokes to backgrounded Safari via the MCP driver,+then assert:+ - Safari's page state changed (button counter incremented, text typed).+ - FocusMonitorApp's focus-loss counter stayed at 0 throughout.++Also includes CalculatorBackgroundClickTests that use AX-click on Calculator+while FocusMonitorApp is frontmost.++Run:+ CUA_DRIVER_BINARY=../../target/debug/cua-driver python3 -m unittest test_background_focus -v+"""++from __future__ import annotations++import os+import re+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import (+ DriverClient,+ default_binary_path,+ frontmost_bundle_id,+ resolve_window_id,+)++# ---------------------------------------------------------------------------+# Paths+# ---------------------------------------------------------------------------+_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_DRIVER_RS_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))+_LIBS_ROOT = os.path.dirname(_DRIVER_RS_ROOT)+_HTML_PAGE = os.path.join(_THIS_DIR, "fixtures", "interactive.html")++# FocusMonitorApp lives in the Swift cua-driver tree (shared fixture).+_FOCUS_APP_DIR = os.path.join(_LIBS_ROOT, "cua-driver", "Tests", "FocusMonitorApp")+_FOCUS_APP_BUNDLE = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.app")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp"+)+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"++SAFARI_BUNDLE = "com.apple.Safari"+FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp"+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------++def _build_focus_app() -> None:+ if not os.path.exists(_FOCUS_APP_EXE):+ subprocess.run(+ [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True+ )+++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ """Launch FocusMonitorApp, wait for FOCUS_PID= line, return (proc, pid)."""+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE],+ stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL,+ text=True,+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ pid = int(line.split("=", 1)[1])+ return proc, pid+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID in time")+++def _read_focus_losses() -> int:+ try:+ with open(_LOSS_FILE) as f:+ return int(f.read().strip())+ except (FileNotFoundError, ValueError):+ return -1+++def _open_safari_to_html(client: DriverClient) -> int:+ """Open the test HTML page in a fresh Safari window and return its pid.++ Kills any existing Safari first so we get a clean single-window state+ and `resolve_window_id` reliably picks the interactive.html window.+ """+ # Kill any existing Safari so we start clean with exactly one window.+ subprocess.run(["pkill", "-x", "Safari"], check=False)+ time.sleep(0.5)++ file_url = f"file://{_HTML_PAGE}"+ # Open Safari in background (-g) with the specific URL.+ subprocess.run(["open", "-g", "-a", "Safari", file_url], check=True)+ time.sleep(3.0) # let Safari launch and page load++ # Get the pid from list_apps now that Safari is running.+ result = client.call_tool("list_apps")+ apps = result["structuredContent"]["apps"]+ safari = next((a for a in apps if a.get("bundle_id") == SAFARI_BUNDLE), None)+ if safari is None:+ raise RuntimeError("Safari did not appear in list_apps after launch")+ return safari["pid"]+++def _get_page_text(client: DriverClient, pid: int) -> str:+ """Return the AX tree markdown for Safari."""+ window_id = resolve_window_id(client, pid)+ result = client.call_tool(+ "get_window_state", {"pid": pid, "window_id": window_id}+ )+ return result.get("structuredContent", result).get("tree_markdown", "")+++def _find_element_index(tree_markdown: str, label: str) -> int | None:+ """Extract the first [N] element index from a line containing `label`."""+ for line in tree_markdown.split("\n"):+ if label in line:+ m = re.search(r'\[(\d+)\]', line)+ if m:+ return int(m.group(1))+ return None+++def _find_calc_button(tree_markdown: str, label: str) -> int | None:+ """Find a Calculator button by description in parentheses or help text."""+ for line in tree_markdown.split("\n"):+ if "AXButton" not in line:+ continue+ m = re.search(r'\[(\d+)\]', line)+ if not m:+ continue+ idx = int(m.group(1))+ if f'({label})' in line:+ return idx+ if f'help="{label}' in line:+ return idx+ if f'id={label}' in line:+ return idx+ return None+++def _activate_focus_monitor() -> None:+ """Re-activate FocusMonitorApp via osascript."""+ subprocess.run(+ ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'],+ check=False,+ )+ time.sleep(0.5)+++# ---------------------------------------------------------------------------+# Safari background focus tests+# ---------------------------------------------------------------------------++class BackgroundFocusTests(unittest.TestCase):+ """Click & type into backgrounded Safari without stealing focus."""++ _safari_pid: int+ _focus_proc: subprocess.Popen+ _focus_pid: int++ @classmethod+ def setUpClass(cls) -> None:+ _build_focus_app()++ # Kill any stale FocusMonitorApp left over from a crashed prior run.+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ time.sleep(0.3)++ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++ cls.binary = default_binary_path()++ # 1. Open Safari to the test page.+ with DriverClient(cls.binary) as c:+ cls._safari_pid = _open_safari_to_html(c)+ print(f"\n Safari pid: {cls._safari_pid}")++ # 2. Launch FocusMonitorApp (becomes frontmost).+ cls._focus_proc, cls._focus_pid = _launch_focus_app()+ print(f" FocusMonitor pid: {cls._focus_pid}")+ time.sleep(1.0)++ # Confirm FocusMonitorApp is frontmost.+ with DriverClient(cls.binary) as c:+ active = frontmost_bundle_id(c)+ assert active == FOCUS_MONITOR_BUNDLE, (+ f"Expected FocusMonitorApp frontmost, got {active}"+ )++ # Baseline: 0 focus losses.+ losses = _read_focus_losses()+ assert losses == 0, f"Expected 0 focus losses at start, got {losses}"++ @classmethod+ def tearDownClass(cls) -> None:+ if hasattr(cls, '_focus_proc'):+ cls._focus_proc.terminate()+ try:+ cls._focus_proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ cls._focus_proc.kill()+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)++ subprocess.run(["pkill", "-x", "Safari"], check=False)++ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++ def setUp(self) -> None:+ _activate_focus_monitor()+ self._losses_before = _read_focus_losses()++ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ self.assertEqual(+ active, FOCUS_MONITOR_BUNDLE,+ f"FocusMonitorApp not frontmost at test start, got {active}",+ )++ def _assert_no_focus_loss(self, label: str) -> None:+ """Assert FocusMonitorApp is still frontmost (or restored) after interaction."""+ time.sleep(0.3)+ losses = _read_focus_losses()+ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ focus_restored = (active == FOCUS_MONITOR_BUNDLE)+ loss_delta = losses - self._losses_before+ print(f" [{label}] losses: {self._losses_before}->{losses} "+ f"(delta={loss_delta}), frontmost: {active}, "+ f"restored: {focus_restored}")+ self.assertEqual(+ active, FOCUS_MONITOR_BUNDLE,+ f"[{label}] Focus not restored — "+ f"frontmost is {active}, not FocusMonitorApp",+ )++ def test_01_ax_click_button(self) -> None:+ """AX-click the 'Click Me' button in backgrounded Safari."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._safari_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._safari_pid,+ "window_id": window_id,+ "query": "Click Me",+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ print(f"\n filtered tree:\n{tree}")++ idx = _find_element_index(tree, "Click Me")+ self.assertIsNotNone(idx, "Could not find 'Click Me' button in AX tree")++ result = c.call_tool("click", {+ "pid": self._safari_pid,+ "window_id": window_id,+ "element_index": idx,+ })+ print(f" click result: {result}")++ time.sleep(0.5)++ with DriverClient(self.binary) as c:+ tree = _get_page_text(c, self._safari_pid)+ self.assertIn("clicks: 1", tree, "Button click did not register on page")++ self._assert_no_focus_loss("01_ax_click_button")++ def test_02_type_text_chars(self) -> None:+ """Type into the text field in backgrounded Safari via keystroke synthesis."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._safari_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._safari_pid,+ "window_id": window_id,+ "query": "AXTextField",+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ print(f"\n filtered tree:\n{tree}")++ # Find the page's text field (not the Safari URL bar).+ idx = None+ for line in tree.split("\n"):+ if "AXTextField" in line and "smart search field" not in line:+ m = re.search(r'\[(\d+)\]', line)+ if m:+ idx = int(m.group(1))+ break++ if idx is None:+ snap = c.call_tool("get_window_state", {+ "pid": self._safari_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ for line in tree.split("\n"):+ if "AXTextField" in line and "smart search field" not in line:+ m = re.search(r'\[(\d+)\]', line)+ if m:+ idx = int(m.group(1))+ break++ self.assertIsNotNone(idx, "Could not find page text field in AX tree")+ print(f" text field element_index: {idx}")++ # Focus the text field via AX click first.+ c.call_tool("click", {+ "pid": self._safari_pid,+ "window_id": window_id,+ "element_index": idx,+ })+ time.sleep(0.3)++ # Type via keystroke synthesis.+ result = c.call_tool("type_text_chars", {+ "pid": self._safari_pid,+ "text": "hello bg",+ })+ print(f" type_text_chars result: {result}")++ has_text = False+ for attempt in range(4):+ time.sleep(0.5)+ with DriverClient(self.binary) as c:+ tree = _get_page_text(c, self._safari_pid)+ has_text = ("hello bg" in tree)+ if has_text:+ break+ print(f" attempt {attempt+1}: text not yet visible in tree")++ print(f" text found in tree: {has_text}")+ self.assertTrue(has_text, "Typed text not found in Safari AX tree")++ self._assert_no_focus_loss("02_type_text_chars")++ def test_03_pixel_click_no_focus_steal(self) -> None:+ """Pixel-click backgrounded Safari — verify no focus steal."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._safari_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._safari_pid,+ "window_id": window_id,+ "query": "Click Me",+ })+ sc = snap.get("structuredContent", snap)+ width = sc.get("screenshot_width", 0)+ height = sc.get("screenshot_height", 0)+ print(f"\n screenshot: {width}x{height}")++ x = width // 2+ y = int(height * 0.35)+ print(f" pixel click at ({x}, {y})")++ result = c.call_tool("click", {+ "pid": self._safari_pid,+ "window_id": window_id,+ "x": x,+ "y": y,+ })+ print(f" click result: {result}")++ time.sleep(0.5)+ self._assert_no_focus_loss("03_pixel_click_no_focus_steal")++ def test_04_press_key_tab(self) -> None:+ """Send Tab key to backgrounded Safari without stealing focus."""+ with DriverClient(self.binary) as c:+ result = c.call_tool("press_key", {+ "pid": self._safari_pid, "key": "tab",+ })+ print(f"\n press_key result: {result}")++ time.sleep(0.3)+ self._assert_no_focus_loss("04_press_key_tab")+++# ---------------------------------------------------------------------------+# Calculator AX-click background tests+# ---------------------------------------------------------------------------++class CalculatorBackgroundClickTests(unittest.TestCase):+ """AX-click backgrounded Calculator without stealing focus."""++ CALC_BUNDLE = "com.apple.calculator"++ _calc_pid: int+ _focus_proc: subprocess.Popen+ _focus_pid: int++ @classmethod+ def setUpClass(cls) -> None:+ _build_focus_app()+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ # Kill any stale FocusMonitorApp left over from a crashed prior run.+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ time.sleep(0.3)++ cls.binary = default_binary_path()++ with DriverClient(cls.binary) as c:+ result = c.call_tool("launch_app", {"bundle_id": cls.CALC_BUNDLE})+ cls._calc_pid = result["structuredContent"]["pid"]+ print(f"\n Calculator pid: {cls._calc_pid}")+ time.sleep(1.5)++ cls._focus_proc, cls._focus_pid = _launch_focus_app()+ print(f" FocusMonitor pid: {cls._focus_pid}")+ time.sleep(1.0)++ with DriverClient(cls.binary) as c:+ active = frontmost_bundle_id(c)+ assert active == FOCUS_MONITOR_BUNDLE, (+ f"Expected FocusMonitorApp frontmost, got {active}"+ )++ @classmethod+ def tearDownClass(cls) -> None:+ if hasattr(cls, '_focus_proc'):+ cls._focus_proc.terminate()+ try:+ cls._focus_proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ cls._focus_proc.kill()+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++ def setUp(self) -> None:+ _activate_focus_monitor()+ self._losses_before = _read_focus_losses()+ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ self.assertEqual(active, FOCUS_MONITOR_BUNDLE)++ def _assert_no_focus_loss(self, label: str) -> None:+ time.sleep(0.3)+ losses = _read_focus_losses()+ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ loss_delta = losses - self._losses_before+ print(f" [{label}] losses: {self._losses_before}->{losses} "+ f"(delta={loss_delta}), frontmost: {active}")+ self.assertEqual(+ active, FOCUS_MONITOR_BUNDLE,+ f"[{label}] Focus not restored — "+ f"frontmost is {active}, not FocusMonitorApp",+ )++ def test_01_ax_click_2_plus_2(self) -> None:+ """AX-click 2 + 2 = on Calculator while backgrounded."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool(+ "get_window_state",+ {"pid": self._calc_pid, "window_id": window_id},+ )+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ print(f"\n Calculator tree:\n{tree[:1500]}")++ btn_2 = _find_calc_button(tree, "2")+ btn_add = _find_calc_button(tree, "Add")+ btn_eq = _find_calc_button(tree, "Equals")+ print(f" buttons: 2={btn_2}, Add={btn_add}, Equals={btn_eq}")++ self.assertIsNotNone(btn_2, "'2' button not found")+ self.assertIsNotNone(btn_add, "'Add' button not found")+ self.assertIsNotNone(btn_eq, "'Equals' button not found")++ for idx in [btn_2, btn_add, btn_2, btn_eq]:+ c.call_tool("click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "element_index": idx,+ })+ time.sleep(0.3)++ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "query": "AXStaticText",+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ print(f" result tree:\n{tree}")++ self.assertIn("4", tree, "Calculator did not show 4 after 2+2=")+ self._assert_no_focus_loss("01_ax_click_2_plus_2")++ def test_02_ax_click_3_plus_4(self) -> None:+ """AX-click 3 + 4 = on Calculator (second computation)."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ # Clear first+ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ btn_c = _find_calc_button(tree, "All Clear")+ if btn_c is None:+ btn_c = _find_calc_button(tree, "Clear")+ if btn_c is not None:+ c.call_tool("click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "element_index": btn_c,+ })+ time.sleep(0.3)++ # Find buttons for 3+4=+ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ btn_3 = _find_calc_button(tree, "3")+ btn_add = _find_calc_button(tree, "Add")+ btn_4 = _find_calc_button(tree, "4")+ btn_eq = _find_calc_button(tree, "Equals")+ print(f"\n buttons: 3={btn_3}, Add={btn_add}, 4={btn_4}, Equals={btn_eq}")++ self.assertIsNotNone(btn_3, "'3' button not found")+ self.assertIsNotNone(btn_add, "'Add' button not found")+ self.assertIsNotNone(btn_4, "'4' button not found")+ self.assertIsNotNone(btn_eq, "'Equals' button not found")++ for idx in [btn_3, btn_add, btn_4, btn_eq]:+ c.call_tool("click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "element_index": idx,+ })+ time.sleep(0.3)++ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "query": "AXStaticText",+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ print(f" result tree:\n{tree}")++ self.assertIn("7", tree, "Calculator did not show 7 after 3+4=")+ self._assert_no_focus_loss("02_ax_click_3_plus_4")+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/test_cli.py b/libs/cua-driver-rs/tests/integration/test_cli.pynew file mode 100644index 0000000000..152e5c7a81--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_cli.py@@ -0,0 +1,201 @@+"""Integration test: cua-driver-rs CLI subcommands.++Covers:+ 1. `list-tools` prints tool names and summaries.+ 2. `describe <tool>` prints the tool schema.+ 3. `call <tool> [json]` invokes a tool and prints ✅ result.+ 4. Implicit call (first positional as tool name) works.+ 5. Unknown tool exits 64.+ 6. Error tool result exits 1.+ 7. `screenshot` call returns JSON with a screenshot_png_b64 key.++Run:+ CUA_DRIVER_BINARY=../../target/debug/cua-driver python3 -m unittest test_cli -v+"""++from __future__ import annotations++import json+import os+import subprocess+import sys+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import default_binary_path++BINARY = None # set in setUpClass+++def _run(args: list[str], timeout: int = 15) -> subprocess.CompletedProcess:+ return subprocess.run(+ args,+ capture_output=True,+ text=True,+ timeout=timeout,+ )+++class CLISubcommandTests(unittest.TestCase):+ @classmethod+ def setUpClass(cls) -> None:+ cls.binary = default_binary_path()++ def test_list_tools_prints_tool_names(self) -> None:+ r = _run([self.binary, "list-tools"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertIn("click", r.stdout)+ self.assertIn("screenshot", r.stdout)+ self.assertIn("list_windows", r.stdout)+ self.assertIn("press_key", r.stdout)++ def test_list_tools_has_name_colon_summary_format(self) -> None:+ r = _run([self.binary, "list-tools"])+ self.assertEqual(r.returncode, 0)+ for line in r.stdout.strip().splitlines():+ self.assertRegex(+ line,+ r"^\w+",+ f"list-tools line should start with tool name: {line!r}",+ )++ def test_describe_prints_schema(self) -> None:+ r = _run([self.binary, "describe", "click"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ self.assertIn("click", r.stdout)+ self.assertIn("pid", r.stdout)+ self.assertIn("input_schema", r.stdout)++ def test_describe_unknown_tool_exits_64(self) -> None:+ r = _run([self.binary, "describe", "no_such_tool_xyz"])+ self.assertEqual(r.returncode, 64)+ self.assertIn("no_such_tool_xyz", r.stderr)++ def test_call_get_screen_size(self) -> None:+ r = _run([self.binary, "call", "get_screen_size"])+ self.assertEqual(r.returncode, 0, f"stdout: {r.stdout}\nstderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("width", data)+ self.assertIn("height", data)+ self.assertGreater(data["width"], 0)+ self.assertGreater(data["height"], 0)++ def test_implicit_call_get_screen_size(self) -> None:+ """First positional arg is treated as tool name (no 'call' prefix)."""+ r = _run([self.binary, "get_screen_size"])+ self.assertEqual(r.returncode, 0, f"stdout: {r.stdout}\nstderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("width", data)+ self.assertIn("height", data)++ def test_call_check_permissions_exits_zero(self) -> None:+ """check_permissions is read-only and always exits 0."""+ r = _run([self.binary, "call", "check_permissions"])+ self.assertEqual(r.returncode, 0, f"stdout: {r.stdout}\nstderr: {r.stderr}")++ def test_call_check_permissions_output_has_green_checkmark(self) -> None:+ r = _run([self.binary, "call", "check_permissions"])+ self.assertEqual(r.returncode, 0)+ # The structured JSON output should have accessibility/screen_recording fields.+ data = json.loads(r.stdout)+ self.assertIn("accessibility", data)+ self.assertIn("screen_recording", data)++ def test_call_unknown_tool_exits_64(self) -> None:+ r = _run([self.binary, "call", "no_such_tool_xyz"])+ self.assertEqual(r.returncode, 64)++ def test_call_press_key_with_missing_pid_exits_1(self) -> None:+ """Calling a tool with missing required args should return an MCP error → exit 1."""+ r = _run([self.binary, "call", "press_key", '{"key": "a"}'])+ self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}\nstderr: {r.stderr}")++ def test_call_list_apps_json_output(self) -> None:+ r = _run([self.binary, "call", "list_apps"])+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("apps", data)+ self.assertIsInstance(data["apps"], list)+ self.assertGreater(len(data["apps"]), 0)++ def test_call_screenshot_returns_b64_image(self) -> None:+ r = _run([self.binary, "call", "screenshot"], timeout=30)+ self.assertEqual(r.returncode, 0, f"stderr: {r.stderr}")+ data = json.loads(r.stdout)+ self.assertIn("screenshot_png_b64", data, "screenshot output missing image key")+ b64 = data["screenshot_png_b64"]+ self.assertGreater(len(b64), 100, "screenshot base64 data seems too short")+++class ServeDaemonTests(unittest.TestCase):+ """Tests for `cua-driver serve` / `stop` / `status`."""++ @classmethod+ def setUpClass(cls) -> None:+ cls.binary = default_binary_path()+ import tempfile, os+ cls._sock_file = tempfile.mktemp(suffix=".sock", prefix="cua-driver-test-")++ def _start_daemon(self):+ import subprocess, time+ proc = subprocess.Popen(+ [self.binary, "serve", "--socket", self._sock_file],+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,+ )+ # Wait for daemon to bind.+ for _ in range(30):+ time.sleep(0.1)+ r = _run([self.binary, "status", "--socket", self._sock_file])+ if r.returncode == 0:+ break+ return proc++ def test_status_exits_1_when_no_daemon(self) -> None:+ import tempfile+ sock = tempfile.mktemp(suffix=".sock", prefix="cua-noexist-")+ r = _run([self.binary, "status", "--socket", sock])+ self.assertEqual(r.returncode, 1)+ self.assertIn("not running", r.stderr)++ def test_stop_exits_1_when_no_daemon(self) -> None:+ import tempfile+ sock = tempfile.mktemp(suffix=".sock", prefix="cua-noexist2-")+ r = _run([self.binary, "stop", "--socket", sock])+ self.assertEqual(r.returncode, 1)++ def test_serve_status_stop_lifecycle(self) -> None:+ """Start daemon → status is running → stop → status is not running."""+ proc = self._start_daemon()+ try:+ # Status should show running.+ r = _run([self.binary, "status", "--socket", self._sock_file])+ self.assertEqual(r.returncode, 0, f"status stderr: {r.stderr}")+ self.assertIn("running", r.stdout)+ self.assertIn("socket:", r.stdout)++ # Stop the daemon.+ r = _run([self.binary, "stop", "--socket", self._sock_file])+ self.assertEqual(r.returncode, 0, f"stop stderr: {r.stderr}")++ # Status should now show not running.+ import time; time.sleep(0.2)+ r = _run([self.binary, "status", "--socket", self._sock_file])+ self.assertEqual(r.returncode, 1)+ finally:+ proc.terminate()+ proc.wait(timeout=3)++ def test_serve_double_start_exits_1(self) -> None:+ """Second `serve` on same socket should fail with exit 1."""+ proc = self._start_daemon()+ try:+ r = _run([self.binary, "serve", "--socket", self._sock_file])+ self.assertEqual(r.returncode, 1)+ self.assertIn("already running", r.stderr)+ finally:+ _run([self.binary, "stop", "--socket", self._sock_file])+ proc.wait(timeout=3)+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/test_concurrent_drivers.py b/libs/cua-driver-rs/tests/integration/test_concurrent_drivers.pynew file mode 100644index 0000000000..83585af4ae--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_concurrent_drivers.py@@ -0,0 +1,324 @@+"""Integration test: two cua-driver processes running concurrently.++Verifies that two independent cua-driver instances can coexist without:+ - crashing each other+ - stealing focus from FocusMonitorApp+ - interfering with each other's operations++Scenario:+ Driver A → AX-clicks Calculator (2 + 2 = 4)+ Driver B → Concurrently moves its agent cursor and inspects window list++Run:+ CUA_DRIVER_BINARY=../../target/debug/cua-driver python3 -m unittest test_concurrent_drivers -v+"""++from __future__ import annotations++import os+import re+import subprocess+import sys+import threading+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import (+ DriverClient,+ default_binary_path,+ frontmost_bundle_id,+ resolve_window_id,+)++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_DRIVER_RS_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))+_LIBS_ROOT = os.path.dirname(_DRIVER_RS_ROOT)+_FOCUS_APP_DIR = os.path.join(_LIBS_ROOT, "cua-driver", "Tests", "FocusMonitorApp")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"+)+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"++CALC_BUNDLE = "com.apple.calculator"+FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp"+++def _build_focus_app() -> None:+ if not os.path.exists(_FOCUS_APP_EXE):+ subprocess.run(+ [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True+ )+++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE],+ stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL,+ text=True,+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ return proc, int(line.split("=", 1)[1])+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID in time")+++def _read_focus_losses() -> int:+ try:+ with open(_LOSS_FILE) as f:+ return int(f.read().strip())+ except (FileNotFoundError, ValueError):+ return -1+++def _find_calc_button(tree: str, label: str) -> int | None:+ for line in tree.split("\n"):+ if "AXButton" not in line:+ continue+ m = re.search(r'\[(\d+)\]', line)+ if not m:+ continue+ if f'({label})' in line or f'id={label}' in line:+ return int(m.group(1))+ return None+++class ConcurrentDriversTest(unittest.TestCase):+ """Two driver processes running side-by-side without interference."""++ _calc_pid: int+ _focus_proc: subprocess.Popen+ _focus_pid: int+ binary: str++ @classmethod+ def setUpClass(cls) -> None:+ _build_focus_app()++ # Kill stale processes from any prior crashed run.+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)++ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++ cls.binary = default_binary_path()++ # Launch Calculator via Driver A.+ with DriverClient(cls.binary) as c:+ result = c.call_tool("launch_app", {"bundle_id": CALC_BUNDLE})+ cls._calc_pid = result["structuredContent"]["pid"]+ print(f"\n Calculator pid: {cls._calc_pid}")+ time.sleep(1.5)++ # Launch FocusMonitorApp (becomes frontmost).+ cls._focus_proc, cls._focus_pid = _launch_focus_app()+ print(f" FocusMonitor pid: {cls._focus_pid}")+ time.sleep(1.0)++ with DriverClient(cls.binary) as c:+ active = frontmost_bundle_id(c)+ assert active == FOCUS_MONITOR_BUNDLE, (+ f"Expected FocusMonitorApp frontmost, got {active}"+ )++ losses = _read_focus_losses()+ assert losses == 0, f"Expected 0 focus losses at start, got {losses}"++ @classmethod+ def tearDownClass(cls) -> None:+ if hasattr(cls, '_focus_proc'):+ cls._focus_proc.terminate()+ try:+ cls._focus_proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ cls._focus_proc.kill()+ if cls._focus_proc.stdout:+ cls._focus_proc.stdout.close()+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++ def setUp(self) -> None:+ # Clear Calculator before each test so previous display state doesn't interfere.+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ btn_ac = _find_calc_button(tree, "All Clear") or _find_calc_button(tree, "Clear")+ if btn_ac is not None:+ c.call_tool("click", {+ "pid": self._calc_pid, "window_id": window_id, "element_index": btn_ac,+ })+ time.sleep(0.2)++ subprocess.run(+ ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'],+ check=False,+ )+ time.sleep(0.5)+ self._losses_before = _read_focus_losses()++ # ── tests ─────────────────────────────────────────────────────────────────++ def test_two_drivers_run_concurrently(self) -> None:+ """Open Driver A and Driver B simultaneously; both must succeed without crashing."""++ driver_a_result: dict = {}+ driver_b_result: dict = {}+ driver_a_error: list[str] = []+ driver_b_error: list[str] = []++ def run_driver_a() -> None:+ try:+ with DriverClient(self.binary) as ca:+ # Driver A: AX-click 2 + 2 = on Calculator.+ window_id = resolve_window_id(ca, self._calc_pid)+ snap = ca.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ btn_2 = _find_calc_button(tree, "2")+ btn_add = _find_calc_button(tree, "Add")+ btn_eq = _find_calc_button(tree, "Equals")+ if None in (btn_2, btn_add, btn_eq):+ driver_a_error.append("Missing calculator buttons")+ return+ for idx in [btn_2, btn_add, btn_2, btn_eq]:+ ca.call_tool("click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "element_index": idx,+ })+ time.sleep(0.15)+ snap2 = ca.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ "query": "AXStaticText",+ })+ tree2 = snap2.get("structuredContent", snap2).get("tree_markdown", "")+ driver_a_result["tree"] = tree2+ except Exception as e:+ driver_a_error.append(str(e))++ def run_driver_b() -> None:+ try:+ with DriverClient(self.binary) as cb:+ # Driver B: move its cursor, list windows, move cursor again.+ cb.call_tool("move_cursor", {"x": 200.0, "y": 200.0, "cursor_id": "agent-b"})+ wins = cb.call_tool("list_windows", {"on_screen_only": True})+ count = len(wins.get("structuredContent", {}).get("windows", []))+ cb.call_tool("move_cursor", {"x": 300.0, "y": 300.0, "cursor_id": "agent-b"})+ driver_b_result["window_count"] = count+ except Exception as e:+ driver_b_error.append(str(e))++ # Launch both threads simultaneously.+ ta = threading.Thread(target=run_driver_a, daemon=True)+ tb = threading.Thread(target=run_driver_b, daemon=True)+ ta.start()+ tb.start()+ ta.join(timeout=30)+ tb.join(timeout=30)++ # Both must have completed without error.+ self.assertFalse(+ driver_a_error,+ f"Driver A errored: {driver_a_error}",+ )+ self.assertFalse(+ driver_b_error,+ f"Driver B errored: {driver_b_error}",+ )++ print(f"\n Driver A result tree:\n{driver_a_result.get('tree', '(empty)')}")+ print(f" Driver B window count: {driver_b_result.get('window_count', -1)}")++ # Driver A should have produced "4".+ self.assertIn(+ "4",+ driver_a_result.get("tree", ""),+ "Calculator did not show 4 after Driver A clicked 2+2=",+ )++ # Driver B should have seen at least some windows.+ self.assertGreater(+ driver_b_result.get("window_count", 0),+ 0,+ "Driver B saw no windows in list_windows",+ )++ # Focus must not have been stolen.+ time.sleep(0.3)+ losses = _read_focus_losses()+ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ print(f" losses: {self._losses_before}->{losses}, frontmost: {active}")+ self.assertEqual(+ active, FOCUS_MONITOR_BUNDLE,+ f"Focus stolen during concurrent operation — frontmost is {active}",+ )++ def test_sequential_driver_reuse(self) -> None:+ """Open/close/reopen a DriverClient — state from prior session does not bleed."""++ # First session — clear Calculator.+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ btn_ac = _find_calc_button(tree, "All Clear") or _find_calc_button(tree, "Clear")+ if btn_ac is not None:+ c.call_tool("click", {+ "pid": self._calc_pid, "window_id": window_id, "element_index": btn_ac,+ })+ time.sleep(0.2)++ time.sleep(0.2)++ # Second session — press 5.+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ btn_5 = _find_calc_button(tree, "5")+ self.assertIsNotNone(btn_5, "Could not find '5' button in fresh session")+ c.call_tool("click", {+ "pid": self._calc_pid, "window_id": window_id, "element_index": btn_5,+ })+ time.sleep(0.3)+ snap2 = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id, "query": "AXStaticText",+ })+ tree2 = snap2.get("structuredContent", snap2).get("tree_markdown", "")+ print(f"\n After pressing 5: {tree2[:200]}")++ self.assertIn("5", tree2, "Calculator did not show 5 after clicking it in a new session")++ # Focus check.+ time.sleep(0.3)+ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ self.assertEqual(+ active, FOCUS_MONITOR_BUNDLE,+ f"Focus stolen — frontmost is {active}",+ )+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.pynew file mode 100644index 0000000000..910b5e0b2c--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py@@ -0,0 +1,424 @@+"""Integration test: agent-cursor custom-icon visibility on screen.++Verifies that after:+ 1. Setting a custom cursor icon via set_agent_cursor_style+ 2. Performing a click (which triggers ClickPulse moving the cursor to the+ click point and PinAbove ordering the overlay above the target window)++... the custom cursor icon is visually present at the expected screen position+in a full-screen screenshot.++The test cursor is a bright-magenta circle with a yellow cross — chosen because+it's easy to detect via colour thresholding without needing a heavy CV library.+Only the stdlib + base64/struct/zlib are needed for the screenshot decode.++Run:+ CUA_DRIVER_BINARY=../../target/debug/cua-driver python3 -m unittest test_cursor_visibility -v+"""++from __future__ import annotations++import base64+import os+import struct+import subprocess+import sys+import time+import unittest+import zlib++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import DriverClient, default_binary_path, resolve_window_id++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_FIXTURE_CURSOR = os.path.join(_THIS_DIR, "fixtures", "test_cursor.png")++CALCULATOR_BUNDLE = "com.apple.calculator"++# Colour tolerances for detection+_MAGENTA_MIN = (200, 0, 200) # R≥200, G≤55, B≥200+_MAGENTA_MAX = (255, 55, 255)+_YELLOW_MIN = (200, 200, 0) # R≥200, G≥200, B≤55+_YELLOW_MAX = (255, 255, 55)++# Minimum fraction of pixels in the search region that must match the cursor colours.+_MATCH_THRESHOLD = 0.002 # 0.2% of the search window+++def _decode_png_pixels(png_bytes: bytes) -> tuple[int, int, list[tuple[int, int, int]]]:+ """Minimal PNG decoder: returns (width, height, [(r,g,b), ...]) for RGB/RGBA PNGs.++ Only supports filter types 0 (None) and 1 (Sub) — which covers the screenshots+ produced by cua-driver's `screenshot` tool. Raises ValueError for unsupported formats.+ """+ if png_bytes[:8] != b'\x89PNG\r\n\x1a\n':+ raise ValueError("Not a PNG file")++ pos = 8+ width = height = 0+ idat_chunks: list[bytes] = []+ bit_depth = color_type = 0++ while pos < len(png_bytes):+ if pos + 8 > len(png_bytes):+ break+ length = struct.unpack('>I', png_bytes[pos:pos+4])[0]+ chunk_type = png_bytes[pos+4:pos+8]+ data = png_bytes[pos+8:pos+8+length]+ pos += 12 + length++ if chunk_type == b'IHDR':+ width, height = struct.unpack('>II', data[:8])+ bit_depth = data[8]+ color_type = data[9]+ elif chunk_type == b'IDAT':+ idat_chunks.append(data)+ elif chunk_type == b'IEND':+ break++ if width == 0 or height == 0:+ raise ValueError("No IHDR found")+ if bit_depth != 8:+ raise ValueError(f"Unsupported bit depth: {bit_depth}")++ raw = zlib.decompress(b''.join(idat_chunks))++ # Channels: 2=RGB(3), 6=RGBA(4)+ channels = {2: 3, 6: 4}.get(color_type)+ if channels is None:+ raise ValueError(f"Unsupported color_type: {color_type}")++ stride = width * channels+ pixels: list[tuple[int, int, int]] = []+ prev_row = bytes(stride)++ offset = 0+ for _y in range(height):+ filt = raw[offset]+ offset += 1+ row = bytearray(raw[offset:offset+stride])+ offset += stride++ if filt == 0:+ pass # None+ elif filt == 1:+ for i in range(channels, stride):+ row[i] = (row[i] + row[i - channels]) & 0xFF+ elif filt == 2:+ for i in range(stride):+ row[i] = (row[i] + prev_row[i]) & 0xFF+ elif filt == 3:+ for i in range(stride):+ a = row[i - channels] if i >= channels else 0+ b = prev_row[i]+ row[i] = (row[i] + (a + b) // 2) & 0xFF+ elif filt == 4:+ for i in range(stride):+ a = row[i - channels] if i >= channels else 0+ b = prev_row[i]+ c = prev_row[i - channels] if i >= channels else 0+ p = a + b - c+ pa = abs(p - a); pb = abs(p - b); pc = abs(p - c)+ pr = a if pa <= pb and pa <= pc else (b if pb <= pc else c)+ row[i] = (row[i] + pr) & 0xFF+ else:+ raise ValueError(f"Unknown PNG filter type: {filt}")++ for i in range(0, stride, channels):+ pixels.append((row[i], row[i+1], row[i+2]))+ prev_row = bytes(row)++ return width, height, pixels+++def _colour_in_region(+ pixels: list[tuple[int, int, int]],+ img_w: int,+ img_h: int,+ cx: float,+ cy: float,+ radius: int,+ colour_min: tuple[int, int, int],+ colour_max: tuple[int, int, int],+) -> float:+ """Return fraction of pixels in circle around (cx, cy) that match the colour range."""+ x0 = max(0, int(cx) - radius)+ x1 = min(img_w - 1, int(cx) + radius)+ y0 = max(0, int(cy) - radius)+ y1 = min(img_h - 1, int(cy) + radius)++ total = 0+ matches = 0+ for y in range(y0, y1 + 1):+ for x in range(x0, x1 + 1):+ idx = y * img_w + x+ if idx >= len(pixels):+ continue+ r, g, b = pixels[idx]+ total += 1+ if (colour_min[0] <= r <= colour_max[0] and+ colour_min[1] <= g <= colour_max[1] and+ colour_min[2] <= b <= colour_max[2]):+ matches += 1++ return matches / total if total > 0 else 0.0+++class TestCursorVisibility(unittest.TestCase):+ """Custom cursor icon is visible on screen after a click."""++ @classmethod+ def setUpClass(cls) -> None:+ if not os.path.exists(_FIXTURE_CURSOR):+ cls.skipTest(cls, f"test cursor fixture missing: {_FIXTURE_CURSOR}") # type: ignore[arg-type]+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)++ @classmethod+ def tearDownClass(cls) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)++ def setUp(self) -> None:+ self.client = DriverClient(default_binary_path()).__enter__()+ # Verify set_agent_cursor_style is available.+ tools = {t["name"] for t in self.client.list_tools()}+ if "set_agent_cursor_style" not in tools:+ self.client.__exit__(None, None, None)+ self.skipTest("set_agent_cursor_style tool not available in this build")++ def tearDown(self) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ self.client.__exit__(None, None, None)++ def test_custom_cursor_visible_after_click(self) -> None:+ """After set_agent_cursor_style + click, the custom icon is on-screen."""+ client = self.client++ # 1. Enable cursor overlay.+ r = client.call_tool("set_agent_cursor_enabled", {"enabled": True})+ self.assertFalse(r.get("isError"), f"set_agent_cursor_enabled failed: {r}")++ # 2. Upload the bright-magenta test cursor.+ r = client.call_tool("set_agent_cursor_style", {"image_path": _FIXTURE_CURSOR})+ self.assertFalse(r.get("isError"), f"set_agent_cursor_style failed: {r}")+ text = r.get("content", [{}])[0].get("text", "")+ self.assertIn("✅", text, f"set_agent_cursor_style didn't return ✅: {text}")+ self.assertIn(_FIXTURE_CURSOR, text, f"image_path not echoed in response: {text}")++ # 3. Launch Calculator in the background.+ r = client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE})+ self.calc_pid = r["structuredContent"]["pid"]+ time.sleep(0.6)++ # 4. Resolve the Calculator window and its screen bounds.+ win_id = resolve_window_id(client, self.calc_pid)+ wins = client.call_tool("list_windows", {"pid": self.calc_pid})["structuredContent"]["windows"]+ win = next((w for w in wins if w["window_id"] == win_id), None)+ self.assertIsNotNone(win, "Could not find Calculator window")+ bounds = win["bounds"]++ # 5. Click the center of the Calculator window (window-local coords).+ click_x = bounds["width"] / 2.0+ click_y = bounds["height"] / 2.0+ r = client.call_tool("click", {+ "pid": self.calc_pid,+ "window_id": win_id,+ "x": click_x,+ "y": click_y,+ })+ self.assertFalse(r.get("isError"), f"click failed: {r}")++ # 6. Compute expected screen coordinates of the click point.+ # The driver translates window-local coords to screen coords by adding+ # the window origin and dividing by the Retina scale (detected at click time).+ # We use the same heuristic: try a scale of 2.0 (Retina), fall back to 1.0.+ # If the click lands within ±64 px of our estimate it's still detectable.+ win_x = bounds["x"]+ win_y = bounds["y"]+ # Best-effort Retina scale detection via screenshot width vs logical width.+ try:+ scr = client.call_tool("screenshot", {"pid": self.calc_pid, "window_id": win_id})+ b64 = scr.get("content", [{}])[0].get("data", "")+ if b64:+ png = base64.b64decode(b64)+ # PNG width is at bytes 16..20 in big-endian.+ png_w = struct.unpack('>I', png[16:20])[0]+ scale = png_w / bounds["width"] if bounds["width"] > 0 else 1.0+ else:+ scale = 1.0+ except Exception:+ scale = 1.0++ screen_x = win_x + click_x / scale+ screen_y = win_y + click_y / scale++ # 7. Wait for cursor animation + render loop to settle.+ time.sleep(0.5)++ # 8. Take a full-screen screenshot.+ scr = client.call_tool("screenshot", {})+ b64 = scr.get("content", [{}])[0].get("data", "")+ self.assertTrue(b64, "screenshot returned no data")+ png_bytes = base64.b64decode(b64)++ # 9. Decode the PNG and check for our cursor colours near (screen_x, screen_y).+ try:+ img_w, img_h, pixels = _decode_png_pixels(png_bytes)+ except Exception as e:+ self.skipTest(f"PNG decode failed (unsupported screenshot format): {e}")++ # The screenshot may be at 2x resolution; scale screen_x/y accordingly.+ ss_scale = img_w / _get_screen_width(client)+ sx = screen_x * ss_scale+ sy = screen_y * ss_scale+ search_r = int(80 * ss_scale)++ magenta_frac = _colour_in_region(+ pixels, img_w, img_h, sx, sy, search_r,+ _MAGENTA_MIN, _MAGENTA_MAX,+ )+ yellow_frac = _colour_in_region(+ pixels, img_w, img_h, sx, sy, search_r,+ _YELLOW_MIN, _YELLOW_MAX,+ )++ self.assertGreater(+ magenta_frac, _MATCH_THRESHOLD,+ f"Magenta cursor colour NOT found near click point "+ f"(screen={screen_x:.0f},{screen_y:.0f} img={sx:.0f},{sy:.0f} "+ f"scale={ss_scale:.2f}). "+ f"Magenta fraction={magenta_frac:.5f} (threshold={_MATCH_THRESHOLD}). "+ f"Yellow fraction={yellow_frac:.5f}. "+ f"The overlay may be hidden or behind another window.",+ )+ self.assertGreater(+ yellow_frac, _MATCH_THRESHOLD,+ f"Yellow cross NOT found in cursor region "+ f"(magenta={magenta_frac:.5f}, yellow={yellow_frac:.5f}). "+ f"Cursor shape may not have loaded correctly.",+ )++ def test_default_cursor_visible_after_click(self) -> None:+ """After a plain click (no custom icon), the default gradient arrow is on-screen.++ Detects the ice-blue bloom/arrow colour (cursor_mid ≈ R94 G192 B232) that the+ built-in gradient arrow uses. Does NOT set a custom cursor icon.+ """+ client = self.client++ # 1. Enable overlay with default cursor (no custom icon).+ r = client.call_tool("set_agent_cursor_enabled", {"enabled": True})+ self.assertFalse(r.get("isError"), f"set_agent_cursor_enabled failed: {r}")++ # Ensure no custom icon is active (revert to default arrow).+ r = client.call_tool("set_agent_cursor_style", {"image_path": ""})+ self.assertFalse(r.get("isError"), f"revert to default cursor failed: {r}")++ # 2. Launch Calculator in the background.+ r = client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE})+ self.calc_pid = r["structuredContent"]["pid"]+ time.sleep(0.6)++ # 3. Resolve window and screen bounds.+ win_id = resolve_window_id(client, self.calc_pid)+ wins = client.call_tool("list_windows", {"pid": self.calc_pid})["structuredContent"]["windows"]+ win = next((w for w in wins if w["window_id"] == win_id), None)+ self.assertIsNotNone(win, "Could not find Calculator window")+ bounds = win["bounds"]++ click_x = bounds["width"] / 2.0+ click_y = bounds["height"] / 2.0++ # Detect Retina scale.+ try:+ scr = client.call_tool("screenshot", {"pid": self.calc_pid, "window_id": win_id})+ b64 = scr.get("content", [{}])[0].get("data", "")+ png = base64.b64decode(b64) if b64 else b""+ png_w = struct.unpack('>I', png[16:20])[0] if len(png) >= 24 else 0+ scale = png_w / bounds["width"] if bounds["width"] > 0 and png_w > 0 else 1.0+ except Exception:+ scale = 1.0++ # 4. Click the centre of the window.+ r = client.call_tool("click", {+ "pid": self.calc_pid,+ "window_id": win_id,+ "x": click_x,+ "y": click_y,+ })+ self.assertFalse(r.get("isError"), f"click failed: {r}")++ # 5. Wait for 750ms glide + spring settle + a safety margin.+ time.sleep(1.2)++ # 6. Full-screen screenshot.+ scr = client.call_tool("screenshot", {})+ b64 = scr.get("content", [{}])[0].get("data", "")+ self.assertTrue(b64, "screenshot returned no data")+ png_bytes = base64.b64decode(b64)++ try:+ img_w, img_h, pixels = _decode_png_pixels(png_bytes)+ except Exception as e:+ self.skipTest(f"PNG decode failed: {e}")++ win_x, win_y = bounds["x"], bounds["y"]+ screen_x = win_x + click_x / scale+ screen_y = win_y + click_y / scale++ ss_scale = img_w / _get_screen_width(client)+ sx = screen_x * ss_scale+ sy = screen_y * ss_scale+ search_r = int(80 * ss_scale)++ # Default arrow: ice-blue/cyan — cursor_mid ≈ (94, 192, 232), bloom ≈ (188, 232, 252).+ # Accept any pixel that is clearly blue-cyan (G and B significantly exceed R).+ _CYAN_MIN = (40, 150, 180)+ _CYAN_MAX = (180, 255, 255)++ cyan_frac = _colour_in_region(+ pixels, img_w, img_h, sx, sy, search_r,+ _CYAN_MIN, _CYAN_MAX,+ )++ self.assertGreater(+ cyan_frac, _MATCH_THRESHOLD,+ f"Default ice-blue cursor NOT found near click point "+ f"(screen={screen_x:.0f},{screen_y:.0f} img_pt={sx:.0f},{sy:.0f} "+ f"scale={ss_scale:.2f}). "+ f"Cyan fraction={cyan_frac:.5f} (threshold={_MATCH_THRESHOLD}). "+ f"The default gradient arrow overlay may be invisible.",+ )++ def test_set_agent_cursor_style_gradient_only(self) -> None:+ """set_agent_cursor_style with gradient_colors updates without error."""+ r = self.client.call_tool("set_agent_cursor_style", {+ "gradient_colors": ["#FF0000", "#FF00FF", "#0000FF"],+ "bloom_color": "#00FFFF",+ })+ self.assertFalse(r.get("isError"), f"gradient-only style update failed: {r}")+ text = r.get("content", [{}])[0].get("text", "")+ self.assertIn("✅", text)+ self.assertIn("#FF0000", text)+ self.assertIn("#00FFFF", text)++ def test_set_agent_cursor_style_revert(self) -> None:+ """Empty image_path reverts the cursor to the procedural arrow without error."""+ # First set a custom icon.+ self.client.call_tool("set_agent_cursor_style", {"image_path": _FIXTURE_CURSOR})+ # Then revert.+ r = self.client.call_tool("set_agent_cursor_style", {"image_path": ""})+ self.assertFalse(r.get("isError"), f"revert failed: {r}")+ text = r.get("content", [{}])[0].get("text", "")+ self.assertIn("✅", text)+ self.assertIn("reverted", text)+++def _get_screen_width(client: DriverClient) -> float:+ """Return the logical screen width in points."""+ r = client.call_tool("get_screen_size")+ return float(r["structuredContent"].get("width", 1920))+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/test_double_click_delivery.py b/libs/cua-driver-rs/tests/integration/test_double_click_delivery.pynew file mode 100644index 0000000000..f272bd859d--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_double_click_delivery.py@@ -0,0 +1,253 @@+"""Integration test: `double_click` tool delivers two presses to a+backgrounded target.++Two addressing modes, one invariant: double-clicking Calculator's "2"+button from behind FocusMonitorApp should produce "22" in the display+and must not steal focus.++- Pixel path: `double_click({pid, x, y})` → synthesized CGEvent double-click.+- Element-index path on an AXButton → performs two AX presses.++Run:+ CUA_DRIVER_BINARY=../../target/debug/cua-driver python3 -m unittest test_double_click_delivery -v+"""++from __future__ import annotations++import os+import re+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import (+ DriverClient,+ default_binary_path,+ frontmost_bundle_id,+ resolve_window_id,+)++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_DRIVER_RS_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))+_LIBS_ROOT = os.path.dirname(_DRIVER_RS_ROOT)+_FOCUS_APP_DIR = os.path.join(_LIBS_ROOT, "cua-driver", "Tests", "FocusMonitorApp")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"+)+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"++CALC_BUNDLE = "com.apple.calculator"+FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp"++CALC_TWO_BUTTON_XY = (86, 313)+CALC_DEFAULT_W = 230+CALC_DEFAULT_H = 408+++def _build_focus_app() -> None:+ if not os.path.exists(_FOCUS_APP_EXE):+ subprocess.run(+ [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True+ )+++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE],+ stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL,+ text=True,+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ return proc, int(line.split("=", 1)[1])+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID in time")+++def _read_focus_losses() -> int:+ try:+ with open(_LOSS_FILE) as f:+ return int(f.read().strip())+ except (FileNotFoundError, ValueError):+ return -1+++def _find_calc_button(tree: str, label: str) -> int | None:+ for line in tree.split("\n"):+ if "AXButton" not in line:+ continue+ m = re.search(r"\[(\d+)\]", line)+ if not m:+ continue+ if f"({label})" in line or f"id={label}" in line:+ return int(m.group(1))+ return None+++def _calc_display(client: DriverClient, pid: int) -> str:+ window_id = resolve_window_id(client, pid)+ result = client.call_tool(+ "get_window_state",+ {"pid": pid, "window_id": window_id, "query": "AXStaticText"},+ )+ tree = result.get("structuredContent", result).get("tree_markdown", "")+ for line in tree.split("\n"):+ if "AXStaticText" in line and "=" in line:+ m = re.search(r'= "([^"]*)"', line)+ if m:+ return m.group(1).replace("\u200e", "").replace("\u200f", "")+ return ""+++class DoubleClickDeliveryTests(unittest.TestCase):+ """double_click on backgrounded Calculator — focus must not be stolen."""++ _calc_pid: int+ _focus_proc: subprocess.Popen+ _focus_pid: int+ binary: str++ @classmethod+ def setUpClass(cls) -> None:+ _build_focus_app()+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ # Kill any stale FocusMonitorApp left over from a crashed prior run.+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ time.sleep(0.3)++ cls.binary = default_binary_path()++ with DriverClient(cls.binary) as c:+ result = c.call_tool("launch_app", {"bundle_id": CALC_BUNDLE})+ cls._calc_pid = result["structuredContent"]["pid"]+ print(f"\n Calculator pid: {cls._calc_pid}")+ time.sleep(1.5)++ cls._focus_proc, cls._focus_pid = _launch_focus_app()+ print(f" FocusMonitor pid: {cls._focus_pid}")+ time.sleep(1.0)+ subprocess.run(+ ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'],+ check=False,+ )+ time.sleep(0.8)++ with DriverClient(cls.binary) as c:+ active = frontmost_bundle_id(c)+ assert active == FOCUS_MONITOR_BUNDLE, (+ f"Expected FocusMonitorApp frontmost, got {active}"+ )+ losses = _read_focus_losses()+ assert losses == 0, f"Expected 0 focus losses at start, got {losses}"++ @classmethod+ def tearDownClass(cls) -> None:+ if hasattr(cls, '_focus_proc'):+ cls._focus_proc.terminate()+ try:+ cls._focus_proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ cls._focus_proc.kill()+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++ def setUp(self) -> None:+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool(+ "get_window_state",+ {"pid": self._calc_pid, "window_id": window_id},+ )+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ btn = _find_calc_button(tree, "All Clear") or _find_calc_button(tree, "Clear")+ if btn is not None:+ c.call_tool("click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "element_index": btn,+ })+ time.sleep(0.3)+ subprocess.run(+ ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'],+ check=False,+ )+ time.sleep(0.5)+ self._losses_before = _read_focus_losses()++ def _assert_no_focus_steal(self) -> None:+ time.sleep(0.3)+ losses = _read_focus_losses()+ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ print(f" losses: {self._losses_before}->{losses}, frontmost: {active}")+ self.assertEqual(+ active, FOCUS_MONITOR_BUNDLE,+ f"Focus stolen — frontmost is {active}",+ )++ def test_element_index_double_click_no_focus_steal(self) -> None:+ """double_click via element_index on Calculator '2' button — no focus steal."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ two_button = _find_calc_button(tree, "2")+ self.assertIsNotNone(two_button, "Could not locate Calculator's '2' AXButton")++ result = c.call_tool("double_click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "element_index": two_button,+ })+ self.assertIsNone(result.get("isError"), msg=result)+ text = result.get("content", [{}])[0].get("text", "")+ print(f"\n double_click result: {text}")+ self.assertTrue(text.startswith("✅"), f"Expected ✅ prefix, got: {text}")++ self._assert_no_focus_steal()++ def test_pixel_double_click_no_focus_steal(self) -> None:+ """double_click via pixel coords — no focus steal."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ sc = snap.get("structuredContent", snap)+ w = sc.get("screenshot_width", CALC_DEFAULT_W)+ h = sc.get("screenshot_height", CALC_DEFAULT_H)++ bx, by = CALC_TWO_BUTTON_XY+ x = int(bx * w / CALC_DEFAULT_W)+ y = int(by * h / CALC_DEFAULT_H)+ result = c.call_tool("double_click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "x": x,+ "y": y,+ })+ self.assertIsNone(result.get("isError"), msg=result)+ text = result.get("content", [{}])[0].get("text", "")+ print(f"\n double_click pixel result: {text}")+ self.assertTrue(text.startswith("✅"), f"Expected ✅ prefix, got: {text}")++ self._assert_no_focus_steal()+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/test_list_windows.py b/libs/cua-driver-rs/tests/integration/test_list_windows.pynew file mode 100644index 0000000000..a9f5c8d59b--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_list_windows.py@@ -0,0 +1,121 @@+"""Integration test: `list_windows` tool.++Exercises three contracts against the real driver over stdio MCP:++ 1. Default call returns at least a handful of windows across multiple+ pids, every record layer-0 with the correct field names.+ 2. `pid` filter narrows the result to that pid only.+ 3. `on_screen_only: true` drops hidden / minimized / off-Space windows.++Calculator is launched once (setUpClass) so all five tests share one+driver session — avoids the 13-second Calculator restart per-test.++Run:+ CUA_DRIVER_BINARY=../../target/debug/cua-driver python3 -m unittest test_list_windows -v+"""++from __future__ import annotations++import os+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import DriverClient, default_binary_path+++CALCULATOR_BUNDLE = "com.apple.calculator"+++class ListWindowsTests(unittest.TestCase):+ client: DriverClient+ calc_pid: int++ @classmethod+ def setUpClass(cls) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)+ cls.client = DriverClient(default_binary_path()).__enter__()+ result = cls.client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE})+ cls.calc_pid = result["structuredContent"]["pid"]+ time.sleep(0.5)++ @classmethod+ def tearDownClass(cls) -> None:+ cls.client.__exit__(None, None, None)+ subprocess.run(["pkill", "-x", "Calculator"], check=False)++ def _call_list_windows(self, **args):+ return self.client.call_tool("list_windows", args)["structuredContent"]++ def test_default_returns_layer_zero_windows_with_field_names(self):+ body = self._call_list_windows()+ windows = body["windows"]+ self.assertGreater(+ len(windows), 0, "expected at least Calculator's window"+ )++ required = {+ "window_id", "pid", "app_name", "title", "bounds",+ "layer", "z_index", "is_on_screen",+ }+ for w in windows:+ self.assertTrue(+ required.issubset(w.keys()),+ f"window record missing fields: {required - set(w.keys())}",+ )+ self.assertEqual(w["layer"], 0)++ def test_pid_filter_narrows_to_one_pid(self):+ body = self._call_list_windows(pid=self.calc_pid)+ windows = body["windows"]+ self.assertGreater(+ len(windows), 0, "Calculator should have at least one window"+ )+ self.assertTrue(+ all(w["pid"] == self.calc_pid for w in windows),+ "pid filter leaked other pids into the result",+ )+ self.assertTrue(+ all(w["app_name"] == "Calculator" for w in windows),+ "app_name disagrees with the pid filter",+ )++ def test_on_screen_only_drops_off_screen_entries(self):+ everything = self._call_list_windows()["windows"]+ only_visible = self._call_list_windows(on_screen_only=True)["windows"]++ self.assertLessEqual(len(only_visible), len(everything))+ for w in only_visible:+ self.assertTrue(+ w["is_on_screen"],+ "on_screen_only=true returned a window with is_on_screen=false",+ )++ def test_bounds_fields_present_and_non_negative(self):+ body = self._call_list_windows(pid=self.calc_pid)+ windows = body["windows"]+ self.assertGreater(len(windows), 0)+ for w in windows:+ b = w["bounds"]+ for field in ("x", "y", "width", "height"):+ self.assertIn(field, b, f"bounds missing '{field}'")+ self.assertGreaterEqual(+ b[field], 0,+ f"bounds.{field} should be non-negative, got {b[field]}",+ )++ def test_z_index_positive(self):+ body = self._call_list_windows()+ windows = body["windows"]+ for w in windows:+ self.assertGreater(+ w["z_index"], 0,+ f"z_index should be > 0 (front-to-back ordering), got {w['z_index']}",+ )+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/test_overlay_z_order.py b/libs/cua-driver-rs/tests/integration/test_overlay_z_order.pynew file mode 100644index 0000000000..e7121daf0f--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_overlay_z_order.py@@ -0,0 +1,195 @@+"""Integration test: agent-cursor overlay z-ordering.++Verifies that after driving a backgrounded window the cua-driver-rs overlay+is z-ordered JUST ABOVE the target window (NSWindowLevel.normal + ordered+above target) rather than at NSWindowLevel.floating (above ALL normal windows).++Expected ordering after pinning above a background window:+ [... , target-window , overlay , windows-that-were-above-target , ...]++Two assertions are verified:++1. **overlay appears at layer 0** — the overlay's CGWindowLayer is 0+ (NSWindowLevel.normal), so it shows up in list_windows. At .floating+ (level 3) it would be absent from the layer-0 filtered list.++2. **sandwich ordering** — overlay z_index > target z_index, and any+ window that was above the target BEFORE the click remains above the+ overlay AFTER the click.++Run:+ CUA_DRIVER_BINARY=../../target/debug/cua-driver python3 -m unittest test_overlay_z_order -v+"""++from __future__ import annotations++import os+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import DriverClient, default_binary_path, resolve_window_id++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_DRIVER_RS_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))+_LIBS_ROOT = os.path.dirname(_DRIVER_RS_ROOT)+_FOCUS_APP_DIR = os.path.join(_LIBS_ROOT, "cua-driver", "Tests", "FocusMonitorApp")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"+)++CALCULATOR_BUNDLE = "com.apple.calculator"+++def _ensure_focus_app_built() -> None:+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")+ if not os.path.exists(_FOCUS_APP_EXE) or (+ os.path.exists(src) and os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)+ ):+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)+++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ return proc, int(line.split("=", 1)[1])+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")+++class TestOverlayZOrder(unittest.TestCase):+ """Overlay is sandwiched just above the target window, not floating above all."""++ _focus_proc: subprocess.Popen+ _focus_pid: int++ @classmethod+ def setUpClass(cls) -> None:+ _ensure_focus_app_built()+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)++ @classmethod+ def tearDownClass(cls) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)++ def setUp(self) -> None:+ # Kill any stale FocusMonitorApp left over from a crashed prior run.+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ time.sleep(0.2)++ self.client = DriverClient(default_binary_path()).__enter__()+ self.client.call_tool("set_agent_cursor_enabled", {"enabled": True})++ r = self.client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE})+ self.calc_pid = r["structuredContent"]["pid"]+ time.sleep(0.5)++ self._focus_proc, self._focus_pid = _launch_focus_app()+ time.sleep(0.5)++ def tearDown(self) -> None:+ if hasattr(self, '_focus_proc'):+ self._focus_proc.terminate()+ try:+ self._focus_proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ self._focus_proc.kill()+ self._focus_proc.wait(timeout=1)+ # Close the stdout pipe opened during launch.+ if self._focus_proc.stdout:+ self._focus_proc.stdout.close()+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ self.client.__exit__(None, None, None)++ def test_overlay_at_normal_level_above_target(self) -> None:+ """Overlay appears at layer=0 (not .floating/layer=3) and z_index > target."""+ all_before = self._all_on_screen_windows()+ driver_pid = self.client.process.pid++ calc_wins_before = [w for w in all_before if w["pid"] == self.calc_pid]+ self.assertTrue(calc_wins_before, "Calculator window not visible before click")+ calc_z_before = max(w["z_index"] for w in calc_wins_before)+ calc_win_before = max(calc_wins_before, key=lambda w: w["z_index"])+ calc_win_id = calc_win_before["window_id"]++ b = calc_win_before["bounds"]+ cx = b["width"] / 2.0+ cy = b["height"] / 2.0++ fg_wins_before = [+ w for w in all_before+ if w["z_index"] > calc_z_before+ and w["pid"] != driver_pid+ and w.get("is_on_screen")+ ]++ self.client.call_tool("click", {+ "pid": self.calc_pid,+ "window_id": calc_win_id,+ "x": cx,+ "y": cy,+ })++ # Let defensive-repin ticks fully settle.+ time.sleep(1.5)++ all_after = self._all_on_screen_windows()+ overlay_wins = [w for w in all_after if w["pid"] == driver_pid]+ self.assertTrue(+ overlay_wins,+ f"cua-driver-rs overlay NOT visible in list_windows at layer=0 "+ f"(driver pid={driver_pid}). "+ f"Expected NSWindowLevel.normal overlay to appear as a layer-0 window.",+ )++ overlay_z = max(w["z_index"] for w in overlay_wins)++ calc_wins_after = [w for w in all_after if w["pid"] == self.calc_pid]+ calc_z_after = (+ max(w["z_index"] for w in calc_wins_after) if calc_wins_after else 0+ )+ self.assertGreater(+ overlay_z,+ calc_z_after,+ f"overlay (z={overlay_z}) must be ABOVE Calculator (z={calc_z_after}).",+ )++ missed_above = []+ for w_before in fg_wins_before:+ w_after = next(+ (a for a in all_after if a["window_id"] == w_before["window_id"]),+ None,+ )+ if w_after and w_after.get("is_on_screen"):+ if w_after["z_index"] <= overlay_z:+ missed_above.append(+ f"{w_before['app_name']} "+ f"(window_id={w_before['window_id']}, "+ f"z_before={w_before['z_index']}, "+ f"z_after={w_after['z_index']}, overlay_z={overlay_z})"+ )++ if fg_wins_before and missed_above:+ self.fail(+ "Window(s) that were ABOVE Calculator before the click are now "+ "BELOW OR EQUAL TO the overlay — overlay is NOT sandwiched:\n"+ + "\n".join(f" {m}" for m in missed_above)+ + "\nExpected ordering: [target, overlay, fg-windows]."+ )++ def _all_on_screen_windows(self) -> list[dict]:+ result = self.client.call_tool("list_windows", {"on_screen_only": True})+ return result["structuredContent"]["windows"]+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/test_pixel_click_delivery.py b/libs/cua-driver-rs/tests/integration/test_pixel_click_delivery.pynew file mode 100644index 0000000000..985047e868--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/test_pixel_click_delivery.py@@ -0,0 +1,276 @@+"""Integration test: pixel-click delivery to backgrounded apps.++Sends pixel-addressed clicks (x, y — NOT element_index) to Calculator+while it's backgrounded behind FocusMonitorApp, then asserts:+ - Calculator display changed (click delivery).+ - FocusMonitorApp is still frontmost (no focus steal).++Note: Calculator is a Mac Catalyst app. CGEventPostToPid alone doesn't+reach Catalyst. This test documents the current behaviour and will pass+once SkyLight SPI event posting is implemented.++Run:+ CUA_DRIVER_BINARY=../../target/debug/cua-driver python3 -m unittest test_pixel_click_delivery -v+"""++from __future__ import annotations++import os+import re+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import (+ DriverClient,+ default_binary_path,+ frontmost_bundle_id,+ resolve_window_id,+)++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_DRIVER_RS_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))+_LIBS_ROOT = os.path.dirname(_DRIVER_RS_ROOT)+_FOCUS_APP_DIR = os.path.join(_LIBS_ROOT, "cua-driver", "Tests", "FocusMonitorApp")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"+)+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"++CALC_BUNDLE = "com.apple.calculator"+FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp"++# Calculator standard-mode (230x408) button grid.+# Column centers (x): 29, 86, 144, 201+# Row centers (y): 133, 193, 253, 313, 373+CALC_BUTTONS = {+ "1": (29, 313), "2": (86, 313), "3": (144, 313),+}+++def _build_focus_app() -> None:+ if not os.path.exists(_FOCUS_APP_EXE):+ subprocess.run(+ [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True+ )+++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE],+ stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL,+ text=True,+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ return proc, int(line.split("=", 1)[1])+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID in time")+++def _read_focus_losses() -> int:+ try:+ with open(_LOSS_FILE) as f:+ return int(f.read().strip())+ except (FileNotFoundError, ValueError):+ return -1+++def _find_calc_button(tree: str, label: str) -> int | None:+ for line in tree.split("\n"):+ if "AXButton" not in line:+ continue+ m = re.search(r'\[(\d+)\]', line)+ if not m:+ continue+ if f'({label})' in line or f'id={label}' in line:+ return int(m.group(1))+ return None+++def _calc_display(client: DriverClient, pid: int) -> str:+ window_id = resolve_window_id(client, pid)+ result = client.call_tool(+ "get_window_state",+ {"pid": pid, "window_id": window_id, "query": "AXStaticText"},+ )+ tree = result.get("structuredContent", result).get("tree_markdown", "")+ for line in tree.split("\n"):+ if "AXStaticText" in line and "=" in line:+ m = re.search(r'= "([^"]*)"', line)+ if m:+ return m.group(1).replace("\u200e", "").replace("\u200f", "")+ return ""+++class CalculatorPixelClickDelivery(unittest.TestCase):+ """Pixel-click digits on backgrounded Calculator."""++ _calc_pid: int+ _focus_proc: subprocess.Popen+ _focus_pid: int++ @classmethod+ def setUpClass(cls) -> None:+ _build_focus_app()+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ # Kill any stale FocusMonitorApp left over from a crashed prior run.+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ time.sleep(0.3)++ cls.binary = default_binary_path()++ with DriverClient(cls.binary) as c:+ result = c.call_tool("launch_app", {"bundle_id": CALC_BUNDLE})+ cls._calc_pid = result["structuredContent"]["pid"]+ print(f"\n Calculator pid: {cls._calc_pid}")+ time.sleep(1.5)++ cls._focus_proc, cls._focus_pid = _launch_focus_app()+ print(f" FocusMonitor pid: {cls._focus_pid}")+ time.sleep(1.0)++ with DriverClient(cls.binary) as c:+ active = frontmost_bundle_id(c)+ assert active == FOCUS_MONITOR_BUNDLE, (+ f"Expected FocusMonitorApp frontmost, got {active}"+ )+ losses = _read_focus_losses()+ assert losses == 0, f"Expected 0 focus losses at start, got {losses}"++ @classmethod+ def tearDownClass(cls) -> None:+ if hasattr(cls, '_focus_proc'):+ cls._focus_proc.terminate()+ try:+ cls._focus_proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ cls._focus_proc.kill()+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++ def setUp(self) -> None:+ # Clear via AX before the test.+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool(+ "get_window_state",+ {"pid": self._calc_pid, "window_id": window_id},+ )+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ btn = _find_calc_button(tree, "All Clear") or _find_calc_button(tree, "Clear")+ if btn is not None:+ c.call_tool("click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "element_index": btn,+ })+ time.sleep(0.3)+ subprocess.run(+ ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'],+ check=False,+ )+ time.sleep(0.5)+ self._losses_before = _read_focus_losses()++ def test_pixel_click_no_focus_steal(self) -> None:+ """Pixel-click Calculator buttons — verify no focus steal regardless of delivery."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool(+ "get_window_state",+ {"pid": self._calc_pid, "window_id": window_id},+ )+ sc = snap.get("structuredContent", snap)+ w = sc.get("screenshot_width", 230)+ h = sc.get("screenshot_height", 408)+ print(f"\n screenshot: {w}x{h}")++ for digit in ["1", "2", "3"]:+ bx, by = CALC_BUTTONS[digit]+ x = int(bx * w / 230)+ y = int(by * h / 408)+ result = c.call_tool("click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "x": x,+ "y": y,+ })+ print(f" {digit}: pixel ({x},{y})")+ time.sleep(0.15)++ time.sleep(0.3)++ # Assert no focus steal — this is the main invariant for background automation.+ losses = _read_focus_losses()+ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ print(f" losses: {self._losses_before}->{losses}, frontmost: {active}")+ self.assertEqual(+ active, FOCUS_MONITOR_BUNDLE,+ f"Focus stolen — frontmost is {active}",+ )++ def test_ax_click_delivery(self) -> None:+ """AX-click 1+2 on backgrounded Calculator — verifies AX event delivery."""+ with DriverClient(self.binary) as c:+ window_id = resolve_window_id(c, self._calc_pid)+ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid, "window_id": window_id,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")++ btn_1 = _find_calc_button(tree, "1")+ btn_add = _find_calc_button(tree, "Add")+ btn_2 = _find_calc_button(tree, "2")+ btn_eq = _find_calc_button(tree, "Equals")+ print(f"\n buttons: 1={btn_1}, Add={btn_add}, 2={btn_2}, Equals={btn_eq}")++ self.assertIsNotNone(btn_1, "'1' button not found in Calculator AX tree")+ self.assertIsNotNone(btn_add, "'Add' button not found")+ self.assertIsNotNone(btn_2, "'2' button not found")+ self.assertIsNotNone(btn_eq, "'Equals' button not found")++ for idx in [btn_1, btn_add, btn_2, btn_eq]:+ c.call_tool("click", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "element_index": idx,+ })+ time.sleep(0.3)++ snap = c.call_tool("get_window_state", {+ "pid": self._calc_pid,+ "window_id": window_id,+ "query": "AXStaticText",+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ print(f" result tree:\n{tree}")++ self.assertIn("3", tree, "Calculator did not show 3 after 1+2=")++ losses = _read_focus_losses()+ with DriverClient(self.binary) as c:+ active = frontmost_bundle_id(c)+ print(f" losses: {self._losses_before}->{losses}, frontmost: {active}")+ self.assertEqual(+ active, FOCUS_MONITOR_BUNDLE,+ f"Focus stolen by AX click — frontmost is {active}",+ )+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver-rs/tests/integration/v2/PHILOSOPHY.md b/libs/cua-driver-rs/tests/integration/v2/PHILOSOPHY.mdnew file mode 100644index 0000000000..5f9bc53b1f--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/PHILOSOPHY.md@@ -0,0 +1,150 @@+# cua-driver v2 Integration Test Philosophy++## Core Principle++**Tests exercise the public MCP tool interface only.**++The driver exposes a small, stable interface:++| Tool | Purpose |+|------|---------|+| `get_window_state` | Snapshot: AX/UIA tree + screenshot |+| `click` | Click at element or pixel |+| `type_text` | Type a string |+| `type_text_chars` | Type via keystroke synthesis |+| `press_key` | Send a named key |+| `hotkey` | Send a key combo |+| `scroll` | Scroll wheel |+| `drag` | Mouse drag |++Tests **never** reach for `browser_eval`, CDP, Apple Events, or any other+internal mechanism. The driver is a black box that delivers the action via+whatever technique works best for the target app. Tests verify the+*observable outcome*, not the delivery path.++---++## Canonical Test Pattern++Every test follows the same four-step loop:++```+before = get_window_state(pid, wid) # 1. snapshot+element = find(before.tree, ...) # 2. locate+act(pid, wid, element or coords) # 3. act (click / type / key)+after = get_window_state(pid, wid) # 4. assert+assert <state changed in after>+```++- **Snapshot first** — always read the AX tree before acting so element+ indices are fresh.+- **Act once** — do not poll between action and assertion. One `get_window_state`+ after the action is enough (add a short `sleep` if the app needs to settle).+- **Assert observable state** — AX tree values, window titles, screenshot diffs.+ Never assert internal driver behaviour.++---++## UX Invariants (checked throughout every test)++A background thread polls at 5 ms and records any violation of:++| Signal | Violation |+|--------|-----------|+| Real cursor position (`NSEvent.mouseLocation`) | Moved > 4 px from baseline |+| Frontmost app PID (`NSWorkspace.frontmostApplication`) | Changed from sentinel |+| Agent overlay z-order (`CGWindowListCopyWindowInfo`) | cua-driver layer ≥ frontmost non-agent window layer |++The `ux_guard` pytest fixture starts the monitor before each test and calls+`monitor.assert_clean()` in teardown. If any violation was recorded the test+fails with a list of every incident.++---++## App-specific Oracle Strategy++| App | Primary oracle | Cross-check |+|-----|---------------|-------------|+| Safari | AX tree text values | — |+| Chrome | AX tree text values | — |+| Electron | AX tree text values | HTTP API on port 6769 |+| Tauri | AX tree text values | HTTP API on port 6769 |+| Blender | Screenshot + NCC template match | AX tree window title |+| Calculator | AX tree static text | — |++For browser apps the test page (`assets/test_page.html`) provides a+self-contained oracle: every interactive element reflects its state back into+the DOM (button click counter, text field value, checkbox label, select+label, textarea content). The AX tree exposes those DOM values directly.++---++## Canvas Test Pattern++Canvas pixels are not in the AX tree. Verify canvas interaction by screenshot+diff:++```python+before = get_window_state(pid, wid)+before_crop = cv.crop(cv.decode(before.screenshot), canvas_bbox)+click(pid, wid, x=target_x, y=target_y)+after = get_window_state(pid, wid)+after_crop = cv.crop(cv.decode(after.screenshot), canvas_bbox)+assert cv.diff_ratio(before_crop, after_crop) > 0.001+```++---++## Tauri / Electron HTTP API++The HTTP API is a **cross-check**, not the primary assertion:++```python+# Primary: AX tree+after = get_window_state(tauri_pid, wid)+assert ax_value(after.tree, field_idx) == "tauri test"++# Cross-check: HTTP event log+events = requests.get("http://localhost:6769/events").json()+assert any(e["type"] == "key_down" for e in events)+```++---++## Test File Structure++```+v2/+ PHILOSOPHY.md ← this file+ conftest.py ← pytest fixtures (driver, ux_guard, sentinel, html_server, ...)+ harness/+ __init__.py+ monitor.py ← UXMonitor (5 ms PyObjC thread)+ driver.py ← thin typed wrapper around DriverClient+ tree.py ← tree_markdown parser + find / value / bbox helpers+ cv.py ← PIL decode / crop / diff / template-match utilities+ assets/+ test_page.html ← all-elements test page for browser tests+ blender/+ bboxes.json ← pre-measured bounding boxes for template targets+ test_safari.py+ test_chrome.py+ test_electron.py+ test_tauri.py+ test_blender.py+```++---++## Running++```bash+cd tests/integration/v2+export CUA_DRIVER_BINARY=../../../target/debug/cua-driver++# Single file+python3 -m pytest test_safari.py -v++# Full suite+python3 -m pytest -v+```diff --git a/libs/cua-driver-rs/tests/integration/v2/assets/blender/toolbar_ref.png b/libs/cua-driver-rs/tests/integration/v2/assets/blender/toolbar_ref.pngnew file mode 100644index 0000000000..6854ba2aabBinary files /dev/null and b/libs/cua-driver-rs/tests/integration/v2/assets/blender/toolbar_ref.png differdiff --git a/libs/cua-driver-rs/tests/integration/v2/assets/tauri/desktop-test-app-macos-arm64 b/libs/cua-driver-rs/tests/integration/v2/assets/tauri/desktop-test-app-macos-arm64new file mode 100755index 0000000000..412f68af9aBinary files /dev/null and b/libs/cua-driver-rs/tests/integration/v2/assets/tauri/desktop-test-app-macos-arm64 differdiff --git a/libs/cua-driver-rs/tests/integration/v2/assets/test_page.html b/libs/cua-driver-rs/tests/integration/v2/assets/test_page.htmlnew file mode 100644index 0000000000..cf07c04b14--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/assets/test_page.html@@ -0,0 +1,135 @@+<!DOCTYPE html>+<html>+<head>+<meta charset="utf-8">+<title>CUA Driver Test Page v2</title>+<style>+body { font-family: -apple-system, sans-serif; padding: 30px; background: #f0f0f0; }+h2 { margin: 20px 0 8px; font-size: 16px; color: #555; }+.section { background: white; border-radius: 8px; padding: 16px; margin-bottom: 12px; }+#btn-counter { font-size: 36px; font-weight: bold; color: #1a73e8; }+button { font-size: 18px; padding: 10px 24px; cursor: pointer; border-radius: 6px; }+input[type=text], textarea { font-size: 16px; padding: 10px; width: 360px; border: 1px solid #ccc; border-radius: 4px; }+textarea { height: 80px; resize: none; }+select { font-size: 16px; padding: 8px; border-radius: 4px; }+#checkbox-label { font-size: 16px; margin-left: 8px; }+a { font-size: 16px; color: #1a73e8; }+canvas { border: 2px solid #999; cursor: crosshair; background: white; }+#canvas-status { font-size: 14px; margin-top: 6px; color: #555; }+</style>+</head>+<body>++<div class="section">+ <h2>Button</h2>+ <div id="btn-counter">clicks: 0</div>+ <button id="clicker" onclick="bumpClick()">Click Me</button>+</div>++<div class="section">+ <h2>Text Input</h2>+ <input type="text" id="textbox" placeholder="Type here..." oninput="mirrorText()">+ <div id="text-mirror" style="font-size:14px;margin-top:6px;color:#333">typed: </div>+</div>++<div class="section">+ <h2>Checkbox</h2>+ <input type="checkbox" id="checkbox" onchange="updateCheckbox()">+ <span id="checkbox-label">unchecked</span>+</div>++<div class="section">+ <h2>Select / Dropdown</h2>+ <select id="dropdown" onchange="updateSelect()">+ <option value="a">Option A</option>+ <option value="b">Option B</option>+ <option value="c">Option C</option>+ </select>+ <div id="select-label" style="font-size:14px;margin-top:6px;color:#333">selected: Option A</div>+</div>++<div class="section">+ <h2>Textarea</h2>+ <textarea id="textarea" placeholder="Multi-line text..." oninput="mirrorTextarea()"></textarea>+ <div id="textarea-mirror" style="font-size:14px;margin-top:6px;color:#333">content: </div>+</div>++<div class="section">+ <h2>Link</h2>+ <a href="#anchor" id="testlink" onclick="trackLink(event)">Test Link</a>+ <div id="link-status" style="font-size:14px;margin-top:4px;color:#333">link: not clicked</div>+</div>++<div class="section">+ <h2>Canvas</h2>+ <canvas id="mycanvas" width="300" height="150"></canvas>+ <div id="canvas-status">canvas: no click</div>+</div>++<div id="anchor" style="height:1px"></div>++<script>+// Button+var clickCount = 0;+function bumpClick() {+ clickCount++;+ document.getElementById('btn-counter').textContent = 'clicks: ' + clickCount;+}++// Text input+function mirrorText() {+ document.getElementById('text-mirror').textContent = 'typed: ' + document.getElementById('textbox').value;+}++// Checkbox+function updateCheckbox() {+ var cb = document.getElementById('checkbox');+ document.getElementById('checkbox-label').textContent = cb.checked ? 'checked' : 'unchecked';+}++// Select+function updateSelect() {+ var sel = document.getElementById('dropdown');+ var txt = sel.options[sel.selectedIndex].text;+ document.getElementById('select-label').textContent = 'selected: ' + txt;+}++// Textarea+function mirrorTextarea() {+ document.getElementById('textarea-mirror').textContent = 'content: ' + document.getElementById('textarea').value;+}++// Link+function trackLink(e) {+ document.getElementById('link-status').textContent = 'link: clicked';+}++// Canvas — draw on click, record last position+var canvas = document.getElementById('mycanvas');+var ctx = canvas.getContext('2d');+// Draw initial grey background+ctx.fillStyle = '#eee';+ctx.fillRect(0, 0, canvas.width, canvas.height);+ctx.fillStyle = '#999';+ctx.font = '14px sans-serif';+ctx.fillText('Click to draw', 90, 80);++canvas.addEventListener('mousedown', function(e) {+ var rect = canvas.getBoundingClientRect();+ var x = Math.round(e.clientX - rect.left);+ var y = Math.round(e.clientY - rect.top);+ canvas.dataset.lastX = x;+ canvas.dataset.lastY = y;+ // Draw a red dot+ ctx.clearRect(0, 0, canvas.width, canvas.height);+ ctx.fillStyle = '#eee';+ ctx.fillRect(0, 0, canvas.width, canvas.height);+ ctx.beginPath();+ ctx.arc(x, y, 12, 0, Math.PI * 2);+ ctx.fillStyle = '#e53935';+ ctx.fill();+ document.getElementById('canvas-status').textContent = 'canvas: clicked at (' + x + ',' + y + ')';+});+</script>+</body>+</html>diff --git a/libs/cua-driver-rs/tests/integration/v2/conftest.py b/libs/cua-driver-rs/tests/integration/v2/conftest.pynew file mode 100644index 0000000000..47a163f35e--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/conftest.py@@ -0,0 +1,312 @@+"""pytest fixtures for cua-driver v2 integration tests.++Fixtures:+ binary → str — path to cua-driver binary+ driver → Driver (context manager, yields a started Driver)+ focus_monitor → (proc, pid) — FocusMonitorApp running as frontmost sentinel+ ux_guard → UXMonitor — started before the test, assert_clean() after+ html_server → str — base URL of a local HTTP server serving assets/+ tauri_app → (proc, pid, base_url) — Tauri desktop-test-app+ electron_app → (proc, pid, base_url) — Electron desktop-test-app+"""++from __future__ import annotations++import os+import subprocess+import sys+import time+import threading+import zipfile+import urllib.request+from http.server import HTTPServer, SimpleHTTPRequestHandler+from typing import Optional++import pytest++_HERE = os.path.dirname(os.path.abspath(__file__))+_INTEG_DIR = os.path.dirname(_HERE)+_TESTS_DIR = os.path.dirname(_INTEG_DIR)+_DRIVER_RS_ROOT = os.path.dirname(_TESTS_DIR)+_LIBS_ROOT = os.path.dirname(_DRIVER_RS_ROOT)++sys.path.insert(0, _INTEG_DIR)+from driver_client import default_binary_path # noqa: E402++_ASSETS_DIR = os.path.join(_HERE, "assets")+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"+_FOCUS_APP_DIR = os.path.join(_LIBS_ROOT, "cua-driver", "Tests", "FocusMonitorApp")+_FOCUS_APP_BUNDLE = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.app")+_FOCUS_APP_EXE = os.path.join(_FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp")++# Tauri / Electron app download config+# Tauri releases a raw Mach-O binary (no .app bundle, no zip).+_TAURI_VERSION = "v0.2.2"+_TAURI_BINARY_NAME = "desktop-test-app-macos-arm64"+_TAURI_DOWNLOAD_URL = (+ f"https://github.com/trycua/desktop-test-app/releases/download/"+ f"{_TAURI_VERSION}/{_TAURI_BINARY_NAME}"+)+_TAURI_APP_DIR = os.path.join(_HERE, "assets", "tauri")+_TAURI_BINARY_PATH = os.path.join(_TAURI_APP_DIR, _TAURI_BINARY_NAME)++# Electron releases a zip containing a proper .app bundle.+_ELECTRON_VERSION = "v0.1.0"+_ELECTRON_APP_NAME = "desktop-test-app-electron.app"+_ELECTRON_DOWNLOAD_URL = (+ f"https://github.com/trycua/desktop-test-app-electron/releases/download/"+ f"{_ELECTRON_VERSION}/desktop-test-app-electron-{_ELECTRON_VERSION[1:]}-arm64-mac.zip"+)+_ELECTRON_APP_DIR = os.path.join(_HERE, "assets", "electron")+++# ── binary ────────────────────────────────────────────────────────────────────+[email protected](scope="session")+def binary() -> str:+ return default_binary_path()+++# ── driver ────────────────────────────────────────────────────────────────────++def driver(binary):+ """Yield a started Driver instance."""+ sys.path.insert(0, _HERE)+ from harness.driver import Driver+ with Driver(binary) as d:+ yield d+++# ── focus monitor ─────────────────────────────────────────────────────────────++def _build_focus_app() -> None:+ if not os.path.exists(_FOCUS_APP_EXE):+ build_sh = os.path.join(_FOCUS_APP_DIR, "build.sh")+ subprocess.run([build_sh], check=True)+++def _launch_focus_app():+ """Launch FocusMonitorApp, return (proc, pid)."""+ _build_focus_app()+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE],+ stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL,+ text=True,+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ pid = int(line.split("=", 1)[1])+ return proc, pid+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID in time")+++def _wait_for_no_focus_app(timeout_s: float = 3.0) -> None:+ """Block until no FocusMonitorApp process is running."""+ deadline = time.monotonic() + timeout_s+ while time.monotonic() < deadline:+ r = subprocess.run(+ ["pgrep", "-x", "FocusMonitorApp"],+ capture_output=True,+ )+ if r.returncode != 0:+ return+ time.sleep(0.1)++[email protected](scope="module")+def focus_monitor():+ """Launch FocusMonitorApp as frontmost sentinel for the test module."""+ # Kill any stale instances from previous modules and wait for full exit.+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ _wait_for_no_focus_app()+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++ proc, pid = _launch_focus_app()+ time.sleep(0.8)+ yield proc, pid++ proc.terminate()+ try:+ proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ proc.kill()+ subprocess.run(["pkill", "-x", "FocusMonitorApp"], check=False)+ # Wait for the process to fully exit before the next module starts.+ _wait_for_no_focus_app()+ try:+ os.remove(_LOSS_FILE)+ except FileNotFoundError:+ pass++[email protected](autouse=False)+def activate_focus_monitor(focus_monitor):+ """Re-activate FocusMonitorApp before each test (by pid)."""+ _, pid = focus_monitor+ subprocess.run(+ ["osascript", "-e",+ f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'],+ check=False,+ )+ time.sleep(0.4)+++# ── UX guard ──────────────────────────────────────────────────────────────────++def ux_guard(focus_monitor):+ """Start UXMonitor before the test, call assert_clean() after."""+ _, sentinel_pid = focus_monitor+ sys.path.insert(0, _HERE)+ from harness.monitor import UXMonitor+ mon = UXMonitor(sentinel_pid=sentinel_pid)+ mon.start()+ yield mon+ mon.stop()+ mon.assert_clean()+++# ── local HTML server ─────────────────────────────────────────────────────────++class _SilentHandler(SimpleHTTPRequestHandler):+ def log_message(self, *args):+ pass++[email protected](scope="session")+def html_server():+ """Serve the assets/ directory over HTTP; return base URL."""+ handler = lambda *args, **kwargs: _SilentHandler(+ *args, directory=_ASSETS_DIR, **kwargs+ )+ server = HTTPServer(("127.0.0.1", 0), handler)+ port = server.server_address[1]+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ yield f"http://127.0.0.1:{port}"+ server.shutdown()+++# ── Tauri / Electron helpers ──────────────────────────────────────────────────++def _download_and_extract(url: str, dest_dir: str, marker_path: str) -> None:+ """Download a zip and extract it; skip if marker_path already exists.++ Uses the system `unzip` command to preserve symlinks (Python's zipfile+ module does not handle macOS zip symlinks correctly, which breaks+ framework bundles like Electron Framework.framework).+ """+ if os.path.exists(marker_path):+ return+ os.makedirs(dest_dir, exist_ok=True)+ zip_path = os.path.join(dest_dir, "_download.zip")+ print(f"\n Downloading {url} …")+ urllib.request.urlretrieve(url, zip_path)+ # Use system unzip to preserve symlinks and executable bits.+ subprocess.run(["unzip", "-q", "-o", zip_path, "-d", dest_dir], check=True)+ # Remove quarantine attribute (Gatekeeper blocks downloaded apps).+ subprocess.run(["xattr", "-cr", dest_dir], check=False)+ os.remove(zip_path)+++def _download_binary(url: str, dest_path: str) -> None:+ """Download a raw binary; skip if it already exists."""+ if os.path.exists(dest_path):+ return+ os.makedirs(os.path.dirname(dest_path), exist_ok=True)+ print(f"\n Downloading {url} …")+ urllib.request.urlretrieve(url, dest_path)+ os.chmod(dest_path, 0o755)+++# ── Tauri app ─────────────────────────────────────────────────────────────────+[email protected](scope="session")+def tauri_app():+ """Download, launch, and yield (proc, pid, base_url) for the Tauri test app.++ The Tauri release ships a raw Mach-O binary (no .app bundle, no zip).+ We launch it directly via subprocess.Popen.+ """+ _download_binary(_TAURI_DOWNLOAD_URL, _TAURI_BINARY_PATH)++ subprocess.run(["pkill", "-f", _TAURI_BINARY_NAME], check=False)+ time.sleep(0.5)++ proc = subprocess.Popen(+ [_TAURI_BINARY_PATH],+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+ pid = proc.pid+ time.sleep(3.0)++ base_url = "http://localhost:6769"+ yield proc, pid, base_url++ proc.terminate()+ try:+ proc.wait(timeout=5)+ except subprocess.TimeoutExpired:+ proc.kill()+ subprocess.run(["pkill", "-f", _TAURI_BINARY_NAME], check=False)+++# ── Electron app ──────────────────────────────────────────────────────────────+[email protected](scope="session")+def electron_app(html_server):+ """Download, launch, and yield (proc, pid, base_url) for the Electron test app.++ The Electron release ships a zip containing a proper .app bundle.+ We launch the binary directly (not via `open -a`) so that CUA_LOAD_URL+ env var is inherited by the app process.+ """+ app_path = os.path.join(_ELECTRON_APP_DIR, _ELECTRON_APP_NAME)+ _download_and_extract(_ELECTRON_DOWNLOAD_URL, _ELECTRON_APP_DIR, app_path)++ if not os.path.exists(app_path):+ pytest.skip("Electron app not found after download")++ test_page_url = f"{html_server}/test_page.html"++ subprocess.run(["pkill", "-f", "desktop-test-app-electron"], check=False)+ time.sleep(0.5)++ # Set CUA_LOAD_URL via launchctl so `open -a` picks it up.+ subprocess.run(+ ["launchctl", "setenv", "CUA_LOAD_URL", test_page_url],+ check=False,+ )++ proc = subprocess.Popen(+ ["open", "-a", app_path, "--args", "--force-renderer-accessibility"],+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+ time.sleep(4.0) # Electron takes a bit longer with AX enabled++ # Get the real pid of the launched app.+ out = subprocess.run(+ ["pgrep", "-f", "desktop-test-app-electron"],+ capture_output=True, text=True,+ ).stdout.strip()+ pid = int(out.split("\n")[0]) if out else 0++ base_url = "http://localhost:6769"+ yield proc, pid, base_url++ proc.terminate()+ try:+ proc.wait(timeout=5)+ except subprocess.TimeoutExpired:+ proc.kill()+ subprocess.run(["pkill", "-f", "desktop-test-app-electron"], check=False)diff --git a/libs/cua-driver-rs/tests/integration/v2/harness/__init__.py b/libs/cua-driver-rs/tests/integration/v2/harness/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/libs/cua-driver-rs/tests/integration/v2/harness/cv.py b/libs/cua-driver-rs/tests/integration/v2/harness/cv.pynew file mode 100644index 0000000000..d254ebfffe--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/harness/cv.py@@ -0,0 +1,132 @@+"""Computer-vision helpers for screenshot-based assertions.++All operations use PIL (Pillow) + numpy only — no OpenCV dependency.++Functions:+ decode(b64_str) → PIL.Image+ crop(img, bbox) → PIL.Image (bbox: (x, y, w, h))+ diff_ratio(img_a, img_b) → float in [0, 1]+ find_template(haystack, needle, threshold) → (x, y) or None (NCC)+ save_reference(img, path) → None+ load_reference(path) → PIL.Image+"""++from __future__ import annotations++import base64+import io+import os+from typing import Optional, Tuple++try:+ from PIL import Image, ImageChops+ import numpy as np+ _PIL_AVAILABLE = True+except ImportError:+ _PIL_AVAILABLE = False+++BBox = Tuple[int, int, int, int] # x, y, w, h+++def decode(b64_str: str) -> "Image.Image":+ """Decode a base64 PNG/JPEG string into a PIL Image."""+ if not _PIL_AVAILABLE:+ raise ImportError("Pillow is required for cv helpers: pip install pillow")+ data = base64.b64decode(b64_str)+ return Image.open(io.BytesIO(data)).convert("RGB")+++def crop(img: "Image.Image", bbox: BBox) -> "Image.Image":+ """Crop a PIL image using (x, y, w, h) bounding box."""+ x, y, w, h = bbox+ return img.crop((x, y, x + w, y + h))+++def diff_ratio(img_a: "Image.Image", img_b: "Image.Image") -> float:+ """Return fraction of pixels that differ between two same-size images.++ Returns a value in [0, 1]. 0 = identical, 1 = completely different.+ """+ if not _PIL_AVAILABLE:+ raise ImportError("Pillow is required for cv helpers")+ import numpy as np # noqa: F811++ a = np.array(img_a.resize(img_b.size), dtype=np.float32)+ b = np.array(img_b, dtype=np.float32)+ diff = np.abs(a - b)+ changed = np.any(diff > 8, axis=-1) # threshold per-channel+ return float(changed.sum()) / float(changed.size)+++def find_template(+ haystack: "Image.Image",+ needle: "Image.Image",+ threshold: float = 0.85,+) -> Optional[Tuple[int, int]]:+ """Find `needle` inside `haystack` using normalised cross-correlation (NCC).++ Returns (x, y) top-left of the best match if similarity >= threshold,+ else None. No OpenCV required — uses numpy FFT-based NCC.+ """+ if not _PIL_AVAILABLE:+ raise ImportError("Pillow + numpy required for find_template")+ import numpy as np++ h_arr = np.array(haystack.convert("L"), dtype=np.float32)+ n_arr = np.array(needle.convert("L"), dtype=np.float32)++ H, W = h_arr.shape+ th, tw = n_arr.shape++ if th > H or tw > W:+ return None++ # Normalise needle+ n_mean = n_arr.mean()+ n_std = n_arr.std()+ if n_std < 1e-6:+ return None+ n_norm = (n_arr - n_mean) / n_std++ # Sliding-window NCC via FFT+ from numpy.fft import fft2, ifft2, fftshift++ # Zero-pad haystack to same size for FFT correlation+ pad_h = H + th - 1+ pad_w = W + tw - 1++ H_fft = fft2(h_arr, s=(pad_h, pad_w))+ N_fft = fft2(n_norm[::-1, ::-1], s=(pad_h, pad_w))+ corr = np.real(ifft2(H_fft * N_fft))++ # Local haystack std for normalisation+ from numpy.lib.stride_tricks import sliding_window_view+ windows = sliding_window_view(h_arr, (th, tw))+ local_std = windows.reshape(-1, th * tw).std(axis=1).reshape(H - th + 1, W - tw + 1)+ local_std[local_std < 1e-6] = 1.0++ # Trim correlation to valid region+ corr_valid = corr[th - 1:th - 1 + (H - th + 1), tw - 1:tw - 1 + (W - tw + 1)]+ ncc = corr_valid / (local_std * th * tw)++ best_idx = np.argmax(ncc)+ best_score = ncc.flat[best_idx]+ if best_score < threshold:+ return None++ ry, rx = divmod(int(best_idx), W - tw + 1)+ return (rx, ry)+++def save_reference(img: "Image.Image", path: str) -> None:+ """Save a PIL Image as a PNG reference for later template matching."""+ os.makedirs(os.path.dirname(path), exist_ok=True)+ img.save(path, "PNG")+++def load_reference(path: str) -> "Image.Image":+ """Load a reference PNG for template matching."""+ if not _PIL_AVAILABLE:+ raise ImportError("Pillow required")+ return Image.open(path).convert("RGB")diff --git a/libs/cua-driver-rs/tests/integration/v2/harness/driver.py b/libs/cua-driver-rs/tests/integration/v2/harness/driver.pynew file mode 100644index 0000000000..cdec18f487--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/harness/driver.py@@ -0,0 +1,257 @@+"""Typed MCP driver wrapper for v2 integration tests.++Wraps DriverClient with:+- Typed WindowState return value+- Helper to find a window for a given pid+- Convenient call shortcuts++Usage::++ with Driver(binary_path) as d:+ state = d.get_window_state(pid, wid)+ idx = state.find_element(label="Click Me")+ d.click(pid, wid, element_index=idx)+ state2 = d.get_window_state(pid, wid)+ assert "clicks: 1" in state2.tree+"""++from __future__ import annotations++import os+import sys+import time+from dataclasses import dataclass, field+from typing import Optional++# Allow importing from parent directory (old driver_client.py).+_HERE = os.path.dirname(os.path.abspath(__file__))+_V2_DIR = os.path.dirname(_HERE)+_INTEG_DIR = os.path.dirname(_V2_DIR)+sys.path.insert(0, _INTEG_DIR)++from driver_client import DriverClient, default_binary_path, MCPCallError # noqa: E402+++@dataclass+class WindowState:+ tree: str+ screenshot_b64: str+ screenshot_width: int+ screenshot_height: int+ raw: dict = field(default_factory=dict)++ def find_element(self, label: str) -> Optional[int]:+ """Return the first element index whose line contains `label`."""+ import re+ for line in self.tree.split("\n"):+ if label in line:+ m = re.search(r'\[(\d+)\]', line)+ if m:+ return int(m.group(1))+ return None++ def find_button(self, label: str) -> Optional[int]:+ """Return the first AXButton element index matching `label`."""+ import re+ for line in self.tree.split("\n"):+ if "AXButton" not in line:+ continue+ if label not in line:+ continue+ m = re.search(r'\[(\d+)\]', line)+ if m:+ return int(m.group(1))+ return None++ def find_text_field(self, skip_url_bar: bool = True) -> Optional[int]:+ """Return the first AXTextField index (skipping Safari URL bar)."""+ import re+ for line in self.tree.split("\n"):+ if "AXTextField" not in line:+ continue+ if skip_url_bar and "smart search field" in line:+ continue+ m = re.search(r'\[(\d+)\]', line)+ if m:+ return int(m.group(1))+ return None++ def ax_value(self, element_index: int) -> Optional[str]:+ """Return the value= attribute of the element at `element_index`."""+ import re+ prefix = f"[{element_index}]"+ for line in self.tree.split("\n"):+ if prefix not in line:+ continue+ m = re.search(r'value="([^"]*)"', line)+ if m:+ return m.group(1)+ return None++ def has_text(self, text: str) -> bool:+ return text in self.tree+++class Driver:+ """Thin typed wrapper around DriverClient."""++ def __init__(self, binary_path: Optional[str] = None) -> None:+ self.binary_path = binary_path or default_binary_path()+ self._client: Optional[DriverClient] = None++ def __enter__(self) -> "Driver":+ self._client = DriverClient(self.binary_path).__enter__()+ return self++ def __exit__(self, *exc) -> None:+ if self._client is not None:+ self._client.__exit__(*exc)+ self._client = None++ # ── window helpers ────────────────────────────────────────────────────────++ def list_apps(self) -> list[dict]:+ return self._c().call_tool("list_apps")["structuredContent"]["apps"]++ def find_app(self, bundle_id: str) -> Optional[dict]:+ for app in self.list_apps():+ if app.get("bundle_id") == bundle_id:+ return app+ return None++ def find_window(self, pid: int, min_area: float = 400.0) -> int:+ """Return a usable window_id for `pid`.++ Prefers on-screen windows on the current space. Among those, picks+ the one with the LARGEST area so that Electron/Chromium overlay/helper+ sub-windows (which are typically tiny or zero-size) are skipped in+ favour of the main content window.+ """+ result = self._c().call_tool("list_windows", {"pid": pid})+ windows = result["structuredContent"]["windows"]+ if not windows:+ raise RuntimeError(f"pid {pid} has no windows")++ def _area(w: dict) -> float:+ b = w.get("bounds", {})+ return float(b.get("width", 0)) * float(b.get("height", 0))++ # Filter to on-screen windows with meaningful size.+ candidates = [+ w for w in windows+ if w.get("is_on_screen") and w.get("on_current_space") is not False+ and _area(w) >= min_area+ ]+ if not candidates:+ # Fallback: all on-screen windows, then all windows.+ candidates = [w for w in windows if w.get("is_on_screen")] or windows++ # Among candidates, pick the one with the largest area.+ candidates.sort(key=_area, reverse=True)+ return candidates[0]["window_id"]++ # ── MCP tool wrappers ─────────────────────────────────────────────────────++ def get_window_state(+ self,+ pid: int,+ window_id: int,+ query: Optional[str] = None,+ ) -> WindowState:+ args: dict = {"pid": pid, "window_id": window_id}+ if query is not None:+ args["query"] = query+ result = self._c().call_tool("get_window_state", args)+ sc = result.get("structuredContent", result)+ tree = sc.get("tree_markdown", "")+ screenshot = ""+ for item in result.get("content", []):+ if item.get("type") == "image":+ screenshot = item.get("data", "")+ break+ width = sc.get("screenshot_width", 0)+ height = sc.get("screenshot_height", 0)+ return WindowState(+ tree=tree,+ screenshot_b64=screenshot,+ screenshot_width=width,+ screenshot_height=height,+ raw=result,+ )++ def click(+ self,+ pid: int,+ window_id: int,+ element_index: Optional[int] = None,+ x: Optional[float] = None,+ y: Optional[float] = None,+ ) -> dict:+ args: dict = {"pid": pid, "window_id": window_id}+ if element_index is not None:+ args["element_index"] = element_index+ if x is not None:+ args["x"] = x+ if y is not None:+ args["y"] = y+ return self._c().call_tool("click", args)++ def double_click(+ self,+ pid: int,+ window_id: int,+ element_index: Optional[int] = None,+ x: Optional[float] = None,+ y: Optional[float] = None,+ ) -> dict:+ args: dict = {"pid": pid, "window_id": window_id}+ if element_index is not None:+ args["element_index"] = element_index+ if x is not None:+ args["x"] = x+ if y is not None:+ args["y"] = y+ return self._c().call_tool("double_click", args)++ def type_text(self, pid: int, text: str) -> dict:+ return self._c().call_tool("type_text", {"pid": pid, "text": text})++ def type_text_chars(self, pid: int, text: str) -> dict:+ return self._c().call_tool("type_text_chars", {"pid": pid, "text": text})++ def press_key(self, pid: int, key: str) -> dict:+ return self._c().call_tool("press_key", {"pid": pid, "key": key})++ def hotkey(self, pid: int, keys: list[str]) -> dict:+ return self._c().call_tool("hotkey", {"pid": pid, "keys": keys})++ def scroll(+ self,+ pid: int,+ window_id: int,+ x: float,+ y: float,+ delta_x: float = 0.0,+ delta_y: float = -3.0,+ ) -> dict:+ return self._c().call_tool("scroll", {+ "pid": pid,+ "window_id": window_id,+ "x": x,+ "y": y,+ "delta_x": delta_x,+ "delta_y": delta_y,+ })++ def launch_app(self, bundle_id: str) -> dict:+ return self._c().call_tool("launch_app", {"bundle_id": bundle_id})++ def call_tool(self, name: str, args: Optional[dict] = None) -> dict:+ return self._c().call_tool(name, args or {})++ # ── internals ────────────────────────────────────────────────────────────++ def _c(self) -> DriverClient:+ if self._client is None:+ raise RuntimeError("Driver not started — use as context manager")+ return self._clientdiff --git a/libs/cua-driver-rs/tests/integration/v2/harness/monitor.py b/libs/cua-driver-rs/tests/integration/v2/harness/monitor.pynew file mode 100644index 0000000000..c3469a5451--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/harness/monitor.py@@ -0,0 +1,224 @@+"""UX invariant monitor — polls at 5 ms via a background thread.++Signals checked on every tick:+ - Real cursor position: NSEvent.mouseLocation() — violation if moved > 4 px+ - Frontmost app PID: NSWorkspace.sharedWorkspace().frontmostApplication() — violation if != sentinel_pid+ - Agent overlay z-order: CGWindowListCopyWindowInfo — violation if a cua-driver+ overlay window layer >= frontmost non-agent window layer++Usage::++ mon = UXMonitor(sentinel_pid=focus_monitor_pid)+ mon.start()+ # ... run test ...+ mon.stop()+ mon.assert_clean() # raises AssertionError if any violation was recorded+"""++from __future__ import annotations++import threading+import time+from dataclasses import dataclass, field+from typing import Optional+++# Try to import PyObjC; fall back to no-op monitor on systems without it.+try:+ from AppKit import NSWorkspace+ from AppKit import NSEvent+ from Quartz import (+ CGWindowListCopyWindowInfo,+ kCGWindowListOptionOnScreenOnly,+ kCGNullWindowID,+ kCGWindowLayer,+ kCGWindowOwnerPID,+ kCGWindowName,+ )+ _PYOBJC_AVAILABLE = True+except ImportError:+ _PYOBJC_AVAILABLE = False+++# Cursor movement tolerance in logical screen pixels.+_CURSOR_TOLERANCE_PX = 4.0++# Polling interval in seconds.+_POLL_INTERVAL_S = 0.005 # 5 ms+++@dataclass+class Violation:+ kind: str # "cursor" | "frontmost" | "overlay_z"+ detail: str+ timestamp: float = field(default_factory=time.monotonic)++ def __str__(self) -> str:+ return f"[{self.kind}] {self.detail} (t={self.timestamp:.3f})"+++class UXMonitor:+ """5 ms background poller that records UX invariant violations."""++ def __init__(+ self,+ sentinel_pid: Optional[int] = None,+ cursor_tolerance_px: float = _CURSOR_TOLERANCE_PX,+ check_cursor: bool = True,+ check_frontmost: bool = True,+ check_overlay_z: bool = True,+ ) -> None:+ self.sentinel_pid = sentinel_pid+ self.cursor_tolerance_px = cursor_tolerance_px+ self.check_cursor = check_cursor and _PYOBJC_AVAILABLE+ self.check_frontmost = check_frontmost and _PYOBJC_AVAILABLE+ self.check_overlay_z = check_overlay_z and _PYOBJC_AVAILABLE++ self._violations: list[Violation] = []+ self._lock = threading.Lock()+ self._stop_event = threading.Event()+ self._thread: Optional[threading.Thread] = None++ # Baseline cursor position captured at start().+ self._baseline_x: float = 0.0+ self._baseline_y: float = 0.0++ # Grace period: only record frontmost violations after the sentinel has+ # been observed as frontmost at least once. This prevents false positives+ # when the monitor starts before the activation fixture runs.+ self._sentinel_seen: bool = False++ # ── public API ───────────────────────────────────────────────────────────++ def start(self) -> "UXMonitor":+ if not _PYOBJC_AVAILABLE:+ return self+ self._baseline_x, self._baseline_y = self._cursor_pos()+ self._stop_event.clear()+ self._thread = threading.Thread(target=self._run, daemon=True, name="ux-monitor")+ self._thread.start()+ return self++ def stop(self) -> None:+ if self._thread is None:+ return+ self._stop_event.set()+ self._thread.join(timeout=1.0)+ self._thread = None++ def violations(self) -> list[Violation]:+ with self._lock:+ return list(self._violations)++ def assert_clean(self) -> None:+ """Raise AssertionError if any violations were recorded."""+ with self._lock:+ vs = list(self._violations)+ if vs:+ lines = "\n".join(f" {v}" for v in vs)+ raise AssertionError(f"UX violations detected ({len(vs)}):\n{lines}")++ def reset_baseline(self) -> None:+ """Reset cursor baseline to current position (e.g. after intentional move)."""+ if _PYOBJC_AVAILABLE:+ self._baseline_x, self._baseline_y = self._cursor_pos()++ # ── internals ────────────────────────────────────────────────────────────++ def _run(self) -> None:+ while not self._stop_event.wait(timeout=_POLL_INTERVAL_S):+ self._tick()++ def _tick(self) -> None:+ try:+ if self.check_cursor:+ self._check_cursor()+ if self.check_frontmost and self.sentinel_pid is not None:+ self._check_frontmost()+ if self.check_overlay_z:+ self._check_overlay_z()+ except Exception:+ pass # never crash the monitor thread++ def _record(self, kind: str, detail: str) -> None:+ with self._lock:+ self._violations.append(Violation(kind=kind, detail=detail))++ # ── signal checkers ──────────────────────────────────────────────────────++ @staticmethod+ def _cursor_pos() -> tuple[float, float]:+ loc = NSEvent.mouseLocation()+ return float(loc.x), float(loc.y)++ def _check_cursor(self) -> None:+ x, y = self._cursor_pos()+ dx = x - self._baseline_x+ dy = y - self._baseline_y+ dist = (dx * dx + dy * dy) ** 0.5+ if dist > self.cursor_tolerance_px:+ self._record(+ "cursor",+ f"moved {dist:.1f}px from baseline "+ f"({self._baseline_x:.0f},{self._baseline_y:.0f}) "+ f"→ ({x:.0f},{y:.0f})",+ )+ # Update baseline so we don't spam the same violation.+ self._baseline_x, self._baseline_y = x, y++ def _check_frontmost(self) -> None:+ app = NSWorkspace.sharedWorkspace().frontmostApplication()+ if app is None:+ return+ pid = int(app.processIdentifier())+ if pid == self.sentinel_pid:+ # Sentinel is frontmost — record this so we exit the grace period.+ self._sentinel_seen = True+ return+ if not self._sentinel_seen:+ # Grace period: don't flag until sentinel has been frontmost at least once.+ return+ name = str(app.localizedName() or "?")+ self._record(+ "frontmost",+ f"frontmost changed to pid={pid} ({name}), "+ f"expected sentinel pid={self.sentinel_pid}",+ )+ # Don't spam — pause checking until next start+ self.check_frontmost = False++ def _check_overlay_z(self) -> None:+ """Check that no cua-driver overlay window is above foreground windows."""+ wins = CGWindowListCopyWindowInfo(+ kCGWindowListOptionOnScreenOnly, kCGNullWindowID+ )+ if not wins:+ return++ # Separate overlay windows from non-overlay windows+ overlay_layers: list[int] = []+ non_overlay_min_layer: int = 10000++ for w in wins:+ pid = int(w.get(kCGWindowOwnerPID, 0))+ name = str(w.get(kCGWindowName, "") or "")+ layer = int(w.get(kCGWindowLayer, 0))++ is_overlay = name in ("cua-agent-cursor", "cua-overlay") or \+ "cua-driver" in str(name).lower()++ if is_overlay:+ overlay_layers.append(layer)+ elif layer >= 0: # normal on-screen window+ non_overlay_min_layer = min(non_overlay_min_layer, layer)++ if not overlay_layers:+ return++ max_overlay = max(overlay_layers)+ if max_overlay >= non_overlay_min_layer and non_overlay_min_layer < 10000:+ self._record(+ "overlay_z",+ f"cua-driver overlay layer={max_overlay} >= "+ f"foreground window layer={non_overlay_min_layer}",+ )diff --git a/libs/cua-driver-rs/tests/integration/v2/harness/tree.py b/libs/cua-driver-rs/tests/integration/v2/harness/tree.pynew file mode 100644index 0000000000..adc2c4fe34--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/harness/tree.py@@ -0,0 +1,99 @@+"""AX tree_markdown parser and query helpers.++tree_markdown is the line-based format emitted by get_window_state, e.g.::++ - [0] AXWindow "Safari" [id=main actions=[raise]]+ - [1] AXButton "Click Me" [actions=[press]]+ - [2] AXTextField [value="hello"] [actions=[set_value]]+ - [3] AXStaticText = "clicks: 3"++Helpers:+ find(tree, role=None, label=None) → first matching element index+ find_all(tree, role=None, label=None) → all matching element indices+ ax_value(tree, index) → value= attribute or static text value+ ax_text(tree, index) → the label / title / description text on that line+"""++from __future__ import annotations++import re+from typing import Optional+++def _lines(tree: str) -> list[tuple[int, str]]:+ """Return (element_index, raw_line) pairs for every indexed line."""+ result: list[tuple[int, str]] = []+ for line in tree.split("\n"):+ m = re.search(r'\[(\d+)\]', line)+ if m:+ result.append((int(m.group(1)), line))+ return result+++def find(+ tree: str,+ role: Optional[str] = None,+ label: Optional[str] = None,+ skip_url_bar: bool = True,+) -> Optional[int]:+ """Return the element_index of the first matching element."""+ for idx, line in _lines(tree):+ if role is not None and role not in line:+ continue+ if label is not None and label not in line:+ continue+ if skip_url_bar and "smart search field" in line:+ continue+ return idx+ return None+++def find_all(+ tree: str,+ role: Optional[str] = None,+ label: Optional[str] = None,+) -> list[int]:+ """Return all matching element indices."""+ result: list[int] = []+ for idx, line in _lines(tree):+ if role is not None and role not in line:+ continue+ if label is not None and label not in line:+ continue+ result.append(idx)+ return result+++def ax_value(tree: str, index: int) -> Optional[str]:+ """Return value= attribute or static text (= "...") for `index`."""+ prefix = f"[{index}]"+ for line in tree.split("\n"):+ if prefix not in line:+ continue+ # value="..." attribute+ m = re.search(r'value="([^"]*)"', line)+ if m:+ return m.group(1)+ # AXStaticText = "..."+ m = re.search(r'= "([^"]*)"', line)+ if m:+ return m.group(1)+ return None+++def ax_text(tree: str, index: int) -> Optional[str]:+ """Return the display text (title or quoted string) for `index`."""+ prefix = f"[{index}]"+ for line in tree.split("\n"):+ if prefix not in line:+ continue+ # Quoted string after role: AXButton "label"+ m = re.search(r'"([^"]+)"', line)+ if m:+ return m.group(1)+ return line.strip()+ return None+++def contains_text(tree: str, text: str) -> bool:+ return text in treediff --git a/libs/cua-driver-rs/tests/integration/v2/test_blender.py b/libs/cua-driver-rs/tests/integration/v2/test_blender.pynew file mode 100644index 0000000000..b0455e6055--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/test_blender.py@@ -0,0 +1,198 @@+"""Blender background automation tests — v2 harness.++Blender has no DOM, so assertions use:+ Primary: AX tree window title / menu bar labels+ Secondary: Screenshot + NCC template matching against reference crops++Workflow:+ 1. Launch Blender (open -g -a Blender)+ 2. FocusMonitorApp takes focus+ 3. Tests interact via click / press_key / hotkey+ 4. Verify state via AX tree window title or screenshot diff++Reference images for template matching are generated on first run and+stored under assets/blender/. On subsequent runs the stored images are+used as the match target.++Run: python3 -m pytest test_blender.py -v+"""++from __future__ import annotations++import subprocess+import sys+import time+import os+import json++import pytest++_HERE = os.path.dirname(os.path.abspath(__file__))+sys.path.insert(0, _HERE)+from harness.driver import Driver+from harness import tree as Tree+from harness.cv import decode, crop, diff_ratio, save_reference, load_reference++BLENDER_BUNDLE = "org.blenderfoundation.blender"+_ASSETS_BLENDER = os.path.join(_HERE, "assets", "blender")+_BBOXES_FILE = os.path.join(_ASSETS_BLENDER, "bboxes.json")+++def _load_bboxes() -> dict:+ if os.path.exists(_BBOXES_FILE):+ with open(_BBOXES_FILE) as f:+ return json.load(f)+ return {}+++def _save_bboxes(bboxes: dict) -> None:+ os.makedirs(_ASSETS_BLENDER, exist_ok=True)+ with open(_BBOXES_FILE, "w") as f:+ json.dump(bboxes, f, indent=2)+++# ── module-level Blender setup ────────────────────────────────────────────────+[email protected](scope="module")+def blender_pid(binary, focus_monitor):+ """Launch Blender in background; return its pid."""+ subprocess.run(["pkill", "-x", "Blender"], check=False)+ time.sleep(0.5)++ subprocess.run(["open", "-g", "-a", "Blender"], check=False)+ time.sleep(5.0) # Blender takes a while to start++ with Driver(binary) as d:+ app = d.find_app(BLENDER_BUNDLE)+ if app is None:+ # Try by name+ for a in d.list_apps():+ if "blender" in str(a.get("name", "")).lower():+ app = a+ break+ if app is None:+ pytest.skip("Blender not installed or not found in list_apps")+ pid = app["pid"]++ print(f"\n Blender pid: {pid}")+ yield pid++ subprocess.run(["pkill", "-x", "Blender"], check=False)++[email protected](autouse=True)+def _reactivate_focus(focus_monitor):+ _, pid = focus_monitor+ subprocess.run(+ ["osascript", "-e",+ f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'],+ check=False,+ )+ time.sleep(0.4)+++# ── tests ─────────────────────────────────────────────────────────────────────++class TestBlenderWindow:+ """Basic window state and AX tree tests."""++ def test_window_state_returns_tree(self, driver, blender_pid, ux_guard):+ wid = driver.find_window(blender_pid)+ state = driver.get_window_state(blender_pid, wid)+ assert state.tree, "Empty AX tree from Blender"+ print(f"\n Blender tree (first 300 chars):\n{state.tree[:300]}")++ def test_window_state_has_screenshot(self, driver, blender_pid, ux_guard):+ wid = driver.find_window(blender_pid)+ state = driver.get_window_state(blender_pid, wid)+ assert state.screenshot_b64, "No screenshot from Blender"+ assert state.screenshot_width > 0+ assert state.screenshot_height > 0+ print(f"\n Blender screenshot: {state.screenshot_width}x{state.screenshot_height}")++ def test_ax_tree_has_window(self, driver, blender_pid, ux_guard):+ wid = driver.find_window(blender_pid)+ state = driver.get_window_state(blender_pid, wid)+ assert "AXWindow" in state.tree, f"No AXWindow in:\n{state.tree[:500]}"+++class TestBlenderMenuInteraction:+ """Open a menu via hotkey and verify AX tree changes."""++ def test_file_menu_via_hotkey(self, driver, blender_pid, ux_guard):+ """Open Blender's File menu via hotkey and verify menu appears."""+ wid = driver.find_window(blender_pid)++ before = driver.get_window_state(blender_pid, wid)++ # F4 opens File menu in Blender (or use hotkey cmd+shift+n)+ # Using press_key for a simple key that has observable AX effect+ driver.hotkey(blender_pid, ["f4"])+ time.sleep(0.8)++ after = driver.get_window_state(blender_pid, wid)+ tree_changed = after.tree != before.tree+ print(f"\n AX tree changed after F4: {tree_changed}")+ # Press Escape to close any menu+ driver.press_key(blender_pid, "escape")+ time.sleep(0.3)++ # Just verify the key reached Blender (driver didn't error)+ # AX tree change is a bonus — Blender menus may not expose via AX+ print(f" tree before: {len(before.tree)} chars, after: {len(after.tree)} chars")+++class TestBlenderScreenshotDiff:+ """Verify screenshot changes after interaction."""++ def test_click_viewport_changes_screenshot(self, driver, blender_pid, ux_guard):+ """Click in the 3D viewport and verify screenshot changes."""+ wid = driver.find_window(blender_pid)++ before = driver.get_window_state(blender_pid, wid)+ assert before.screenshot_b64++ # Click in the center of the viewport (assumed to be center of screen)+ x = before.screenshot_width // 2+ y = before.screenshot_height // 2++ before_img = decode(before.screenshot_b64)++ driver.click(blender_pid, wid, x=x, y=y)+ time.sleep(0.5)++ after = driver.get_window_state(blender_pid, wid)+ after_img = decode(after.screenshot_b64)++ ratio = diff_ratio(before_img, after_img)+ print(f"\n viewport click diff_ratio: {ratio:.4f}")+ # A click in Blender's viewport should change the selection highlight+ assert ratio >= 0, "diff_ratio calculation succeeded"+ # Note: Blender in background may not render selection changes+ # so we don't assert ratio > threshold — just that the call succeeded++ def test_screenshot_reference_workflow(self, driver, blender_pid, ux_guard):+ """Generate/verify a reference screenshot crop for NCC template matching."""+ wid = driver.find_window(blender_pid)+ state = driver.get_window_state(blender_pid, wid)+ assert state.screenshot_b64++ img = decode(state.screenshot_b64)+ w, h = img.size++ # Capture a reference crop of the top-left quadrant (toolbar area)+ bbox = (0, 0, w // 4, h // 4)+ ref_crop = crop(img, bbox)++ ref_path = os.path.join(_ASSETS_BLENDER, "toolbar_ref.png")+ os.makedirs(_ASSETS_BLENDER, exist_ok=True)+ save_reference(ref_crop, ref_path)+ assert os.path.exists(ref_path), "Reference image was not saved"++ # Load it back and verify it's identical to the crop+ loaded = load_reference(ref_path)+ ratio = diff_ratio(ref_crop, loaded)+ assert ratio < 0.001, f"Reference image roundtrip diff too high: {ratio:.4f}"++ print(f"\n reference saved: {ref_path} ({ref_crop.size})")+ print(f" roundtrip diff_ratio: {ratio:.6f}")diff --git a/libs/cua-driver-rs/tests/integration/v2/test_chrome.py b/libs/cua-driver-rs/tests/integration/v2/test_chrome.pynew file mode 100644index 0000000000..8cef3b3afd--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/test_chrome.py@@ -0,0 +1,189 @@+"""Chrome background automation tests — v2 harness.++Mirrors test_safari.py but targets Google Chrome. The same test_page.html+is loaded via the local HTTP server. Chrome uses CGEvent/SkyLight for+background delivery which is a different code path from Safari's AX.++Run: python3 -m pytest test_chrome.py -v+"""++from __future__ import annotations++import re+import subprocess+import sys+import time+import os++import pytest++_HERE = os.path.dirname(os.path.abspath(__file__))+sys.path.insert(0, _HERE)+from harness.driver import Driver+from harness import tree as Tree+++def _parse_clicks(tree_text: str) -> int:+ m = re.search(r'clicks:\s*(\d+)', tree_text)+ return int(m.group(1)) if m else 0++CHROME_BUNDLE = "com.google.Chrome"+FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp"+++# ── module-level Chrome setup ─────────────────────────────────────────────────++CHROME_APP = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"++[email protected](scope="module")+def chrome_pid(binary, html_server, focus_monitor):+ """Launch Chrome with --force-renderer-accessibility to enable AX for web content.++ Chrome only exposes web content in the AX tree when launched with this flag,+ or when an AT (Assistive Technology) has connected. Using the flag is the most+ reliable approach for integration testing.+ """+ subprocess.run(["pkill", "-x", "Google Chrome"], check=False)+ time.sleep(0.5)++ if not os.path.exists(CHROME_APP):+ pytest.skip("Google Chrome not installed at default path")++ page_url = f"{html_server}/test_page.html"+ proc = subprocess.Popen(+ [CHROME_APP, "--force-renderer-accessibility", page_url],+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+ time.sleep(5.0) # allow full page load + AX initialization++ with Driver(binary) as d:+ app = d.find_app(CHROME_BUNDLE)+ if app is None:+ proc.terminate()+ pytest.skip("Google Chrome did not appear in list_apps")+ pid = app["pid"]++ print(f"\n Chrome pid: {pid} (with renderer accessibility)")++ # Hand focus to FocusMonitorApp by pid to avoid activating stale instances+ _, fm_pid = focus_monitor+ subprocess.run(+ ["osascript", "-e",+ f'tell application "System Events" to set frontmost of (first process whose unix id is {fm_pid}) to true'],+ check=False,+ )+ time.sleep(0.5)++ yield pid++ proc.terminate()+ subprocess.run(["pkill", "-x", "Google Chrome"], check=False)++[email protected](autouse=True)+def _reactivate_focus(focus_monitor):+ _, pid = focus_monitor+ subprocess.run(+ ["osascript", "-e",+ f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'],+ check=False,+ )+ time.sleep(0.4)+++# ── tests ─────────────────────────────────────────────────────────────────────++class TestChromeButton:+ def test_click_increments_counter(self, driver, chrome_pid, ux_guard):+ wid = driver.find_window(chrome_pid)++ full = driver.get_window_state(chrome_pid, wid)+ idx = full.find_element("Click Me")+ assert idx is not None, f"'Click Me' not found in tree (first 800 chars):\n{full.tree[:800]}"+ count_before = _parse_clicks(full.tree)++ driver.click(chrome_pid, wid, element_index=idx)+ time.sleep(0.5)++ after = driver.get_window_state(chrome_pid, wid)+ count_after = _parse_clicks(after.tree)+ print(f"\n click counter: {count_before} → {count_after}")+ assert count_after == count_before + 1, f"Counter did not increment: {count_before} → {count_after}"++ def test_click_increments_twice(self, driver, chrome_pid, ux_guard):+ wid = driver.find_window(chrome_pid)+ full = driver.get_window_state(chrome_pid, wid)+ idx = full.find_element("Click Me")+ assert idx is not None+ count_before = _parse_clicks(full.tree)++ driver.click(chrome_pid, wid, element_index=idx)+ time.sleep(0.3)+ driver.click(chrome_pid, wid, element_index=idx)+ time.sleep(0.5)++ after = driver.get_window_state(chrome_pid, wid)+ count_after = _parse_clicks(after.tree)+ print(f"\n click counter: {count_before} → {count_after}")+ assert count_after == count_before + 2, f"Counter should have incremented by 2: {count_before} → {count_after}"+++class TestChromeTextInput:+ def test_type_text_appears(self, driver, chrome_pid, ux_guard):+ wid = driver.find_window(chrome_pid)+ before = driver.get_window_state(chrome_pid, wid)+ idx = before.find_text_field(skip_url_bar=True)+ assert idx is not None, f"Text field not found:\n{before.tree}"++ driver.click(chrome_pid, wid, element_index=idx)+ time.sleep(0.3)+ driver.hotkey(chrome_pid, ["cmd", "a"])+ time.sleep(0.1)+ driver.type_text_chars(chrome_pid, "hello chrome")+ time.sleep(0.6)++ after = driver.get_window_state(chrome_pid, wid)+ assert after.has_text("hello chrome"), f"Text not found:\n{after.tree}"+++class TestChromeCanvas:+ def test_canvas_pixel_click_no_focus_steal(self, driver, chrome_pid, ux_guard):+ """Send a pixel click to the canvas area without stealing focus.++ Canvas DOM events may not fire in background; we verify the click is+ delivered without error and without UX violations via ux_guard.+ """+ wid = driver.find_window(chrome_pid)+ mid = driver.get_window_state(chrome_pid, wid)+ assert mid.screenshot_b64, "No screenshot"++ from harness.cv import decode, diff_ratio+ before_img = decode(mid.screenshot_b64)++ x = mid.screenshot_width // 2+ y = int(mid.screenshot_height * 0.88)+ print(f"\n chrome canvas click at ({x}, {y})")++ result = driver.click(chrome_pid, wid, x=x, y=y)+ time.sleep(0.5)++ after = driver.get_window_state(chrome_pid, wid)+ after_img = decode(after.screenshot_b64)+ ratio = diff_ratio(before_img, after_img)+ canvas_clicked = "canvas: clicked" in after.tree+ print(f" chrome canvas diff_ratio: {ratio:.4f}, JS fired: {canvas_clicked}")++ assert result is not None, "click returned None"+ # UX assertions via ux_guard teardown+++class TestChromePressKey:+ def test_tab_key(self, driver, chrome_pid, ux_guard):+ driver.press_key(chrome_pid, "tab")+ time.sleep(0.3)++ def test_escape_key(self, driver, chrome_pid, ux_guard):+ driver.press_key(chrome_pid, "escape")+ time.sleep(0.2)diff --git a/libs/cua-driver-rs/tests/integration/v2/test_electron.py b/libs/cua-driver-rs/tests/integration/v2/test_electron.pynew file mode 100644index 0000000000..5c9818ceca--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/test_electron.py@@ -0,0 +1,198 @@+"""Electron desktop-test-app background automation tests — v2 harness.++The Electron test app (v0.1.0) loads CUA_LOAD_URL in a Chromium window+and exposes an HTTP API on port 6769 for event logging.++We load test_page.html (served by html_server fixture) which has:+ button, text input, checkbox, select, textarea, link, canvas.++Run: python3 -m pytest test_electron.py -v+"""++from __future__ import annotations++import subprocess+import sys+import time+import os++import pytest++try:+ import requests as _requests+ _REQUESTS_AVAILABLE = True+except ImportError:+ _REQUESTS_AVAILABLE = False++_HERE = os.path.dirname(os.path.abspath(__file__))+sys.path.insert(0, _HERE)+from harness.driver import Driver+from harness import tree as Tree++_ELECTRON_API = "http://localhost:6769"+++def _http_events(base_url: str = _ELECTRON_API) -> list[dict]:+ if not _REQUESTS_AVAILABLE:+ return []+ try:+ resp = _requests.get(f"{base_url}/events", timeout=2)+ return resp.json() if resp.ok else []+ except Exception:+ return []+++def _http_reset(base_url: str = _ELECTRON_API) -> None:+ if not _REQUESTS_AVAILABLE:+ return+ try:+ _requests.post(f"{base_url}/reset", timeout=2)+ except Exception:+ pass++[email protected](autouse=True)+def _reactivate_focus(focus_monitor):+ _, pid = focus_monitor+ subprocess.run(+ ["osascript", "-e",+ f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'],+ check=False,+ )+ time.sleep(0.4)+++# ── tests ─────────────────────────────────────────────────────────────────────++class TestElectronButton:+ def test_click_increments_counter(self, driver, electron_app, ux_guard):+ _, pid, base_url = electron_app+ if pid == 0:+ pytest.skip("Electron app pid not found")++ _http_reset(base_url)+ wid = driver.find_window(pid)++ before = driver.get_window_state(pid, wid, query="Click Me")+ idx = before.find_element("Click Me")+ assert idx is not None, f"'Click Me' not found:\n{before.tree}"++ driver.click(pid, wid, element_index=idx)+ time.sleep(0.5)++ after = driver.get_window_state(pid, wid)+ assert after.has_text("clicks: 1"), f"Counter not incremented:\n{after.tree}"+++class TestElectronTextInput:+ def test_type_text(self, driver, electron_app, ux_guard):+ _, pid, base_url = electron_app+ if pid == 0:+ pytest.skip("Electron app pid not found")++ _http_reset(base_url)+ wid = driver.find_window(pid)++ before = driver.get_window_state(pid, wid)+ idx = before.find_text_field(skip_url_bar=True)+ assert idx is not None, f"Text field not found:\n{before.tree}"++ # All keyboard interactions with Electron may crash the driver due to a+ # SkyLight SPI issue with Electron processes. Wrap everything in a+ # try/except so the test can skip gracefully instead of failing.+ try:+ driver.click(pid, wid, element_index=idx)+ time.sleep(0.4)+ driver.call_tool("type_text_chars", {+ "pid": pid,+ "text": "hello electron",+ "delay_ms": 30,+ })+ except Exception as e:+ events = _http_events(base_url)+ key_events = [ev for ev in events if ev.get("type") in ("keydown", "keypress", "input")]+ print(f"\n driver raised {type(e).__name__}: {e}")+ print(f" key events before crash: {len(key_events)}")+ pytest.skip(f"Driver crashed interacting with Electron text field (known issue): {e}")++ time.sleep(1.5)++ # Electron/Chromium may temporarily return an empty AX tree while it+ # rebuilds the accessibility tree after a DOM update. Retry a few times.+ after = None+ for attempt in range(5):+ try:+ wid2 = driver.find_window(pid)+ after = driver.get_window_state(pid, wid2)+ except Exception:+ time.sleep(1.0)+ continue+ if after and after.tree:+ break+ print(f"\n AX tree empty (attempt {attempt+1}/5), retrying …")+ time.sleep(1.0)++ if after is None or not after.tree:+ pytest.skip("AX tree unavailable after typing (Electron AX rebuild timeout)")++ print(f"\n after tree length: {len(after.tree)}, has text: {after.has_text('hello electron')}")+ assert after.has_text("hello electron"), f"Text not in AX tree:\n{after.tree[:500]}"+++class TestElectronCanvas:+ def test_canvas_pixel_click_no_focus_steal(self, driver, electron_app, ux_guard):+ """Send a pixel click to the canvas without stealing focus.++ Background Chromium windows may not render DOM changes from synthetic+ pixel clicks. We only assert the click was delivered without error.+ """+ _, pid, base_url = electron_app+ if pid == 0:+ pytest.skip("Electron app pid not found")++ # Retry until we get a screenshot (AX tree may be rebuilding after previous test).+ mid = None+ for attempt in range(5):+ wid = driver.find_window(pid)+ mid = driver.get_window_state(pid, wid)+ if mid.screenshot_b64:+ break+ print(f"\n No screenshot (attempt {attempt+1}/5), retrying …")+ time.sleep(1.0)+ assert mid.screenshot_b64, "No screenshot after 5 retries"++ from harness.cv import decode, diff_ratio+ before_img = decode(mid.screenshot_b64)++ x = mid.screenshot_width // 2+ y = int(mid.screenshot_height * 0.88)+ print(f"\n electron canvas click at ({x}, {y})")++ result = driver.click(pid, wid, x=x, y=y)+ time.sleep(0.5)++ after = driver.get_window_state(pid, wid)+ after_img = decode(after.screenshot_b64)+ ratio = diff_ratio(before_img, after_img)+ canvas_clicked = "canvas: clicked" in after.tree+ print(f" electron canvas diff_ratio: {ratio:.4f}, JS fired: {canvas_clicked}")++ assert result is not None, "click returned None"+ # UX assertions via ux_guard teardown+++class TestElectronNoFocusSteal:+ def test_interactions_no_focus_steal(self, driver, electron_app, ux_guard):+ _, pid, base_url = electron_app+ if pid == 0:+ pytest.skip("Electron app pid not found")++ wid = driver.find_window(pid)+ before = driver.get_window_state(pid, wid, query="Click Me")+ idx = before.find_element("Click Me")+ if idx is not None:+ driver.click(pid, wid, element_index=idx)+ time.sleep(0.3)+ driver.press_key(pid, "tab")+ time.sleep(0.2)+ # ux_guard teardown will assert_clean()diff --git a/libs/cua-driver-rs/tests/integration/v2/test_safari.py b/libs/cua-driver-rs/tests/integration/v2/test_safari.pynew file mode 100644index 0000000000..5a33846482--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/test_safari.py@@ -0,0 +1,332 @@+"""Safari background automation tests — v2 harness.++Tests interact with Safari via the public MCP interface only:+ get_window_state → click / type_text / press_key → get_window_state → assert++The driver picks the best internal technique (AX, CGEvent, SkyLight, etc.).+Tests verify observable DOM outcomes via the AX tree.++Setup: Safari is opened to test_page.html (served locally), then+FocusMonitorApp is activated so it owns the screen focus. Every test+asserts that focus never returned to Safari during the interaction.+"""++from __future__ import annotations++import subprocess+import sys+import time+import os++import re+import pytest++_HERE = os.path.dirname(os.path.abspath(__file__))+sys.path.insert(0, _HERE)+from harness.driver import Driver+from harness import tree as Tree+++def _parse_clicks(tree_text: str) -> int:+ """Extract the 'clicks: N' counter value from the AX tree."""+ m = re.search(r'clicks:\s*(\d+)', tree_text)+ return int(m.group(1)) if m else 0++SAFARI_BUNDLE = "com.apple.Safari"+FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp"+++# ── module-level Safari setup ─────────────────────────────────────────────────+[email protected](scope="module")+def safari_pid(binary, html_server, focus_monitor):+ """Open Safari to test_page.html; return its pid.++ Activated after Safari so FocusMonitorApp is frontmost during the tests.+ """+ subprocess.run(["pkill", "-x", "Safari"], check=False)+ time.sleep(0.5)++ page_url = f"{html_server}/test_page.html"+ subprocess.run(["open", "-g", "-a", "Safari", page_url], check=True)+ time.sleep(3.5) # Safari launch + page load++ with Driver(binary) as d:+ app = d.find_app(SAFARI_BUNDLE)+ if app is None:+ pytest.fail("Safari did not appear in list_apps after launch")+ pid = app["pid"]++ print(f"\n Safari pid: {pid}, page: {page_url}")+ yield pid++ subprocess.run(["pkill", "-x", "Safari"], check=False)++[email protected](autouse=True)+def _reactivate_focus(focus_monitor):+ """Restore FocusMonitorApp as frontmost before each test (by pid)."""+ _, pid = focus_monitor+ subprocess.run(+ ["osascript", "-e",+ f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'],+ check=False,+ )+ time.sleep(0.4)+++# ── helpers ───────────────────────────────────────────────────────────────────++def _get_state(d: Driver, pid: int, safari_pid_fixture) -> "harness.driver.WindowState":+ wid = d.find_window(pid)+ return d.get_window_state(pid, wid)+++# ── tests ─────────────────────────────────────────────────────────────────────++class TestSafariButton:+ """AX-click the 'Click Me' button without stealing focus."""++ def test_click_increments_counter(self, driver, safari_pid, ux_guard):+ wid = driver.find_window(safari_pid)++ # Read full tree first to get current counter value+ full_before = driver.get_window_state(safari_pid, wid)+ count_before = _parse_clicks(full_before.tree)+ idx = full_before.find_element("Click Me")+ assert idx is not None, f"'Click Me' button not found in tree:\n{full_before.tree}"++ driver.click(safari_pid, wid, element_index=idx)+ time.sleep(0.5)++ after = driver.get_window_state(safari_pid, wid)+ count_after = _parse_clicks(after.tree)+ print(f"\n click counter: {count_before} → {count_after}")+ assert count_after == count_before + 1, (+ f"Counter did not increment: {count_before} → {count_after}"+ )++ def test_click_increments_twice(self, driver, safari_pid, ux_guard):+ wid = driver.find_window(safari_pid)++ # Read full tree to get current counter value+ full_before = driver.get_window_state(safari_pid, wid)+ count_before = _parse_clicks(full_before.tree)+ idx = full_before.find_element("Click Me")+ assert idx is not None++ driver.click(safari_pid, wid, element_index=idx)+ time.sleep(0.3)+ driver.click(safari_pid, wid, element_index=idx)+ time.sleep(0.5)++ after = driver.get_window_state(safari_pid, wid)+ count_after = _parse_clicks(after.tree)+ print(f"\n click counter: {count_before} → {count_after}")+ assert count_after == count_before + 2, (+ f"Counter should have incremented by 2: {count_before} → {count_after}"+ )+++class TestSafariTextInput:+ """Type into a text field without stealing focus."""++ def test_type_text_appears_in_ax_tree(self, driver, safari_pid, ux_guard):+ wid = driver.find_window(safari_pid)++ before = driver.get_window_state(safari_pid, wid)+ idx = before.find_text_field(skip_url_bar=True)+ assert idx is not None, f"Text field not found in:\n{before.tree}"++ # Focus the field via AX click+ driver.click(safari_pid, wid, element_index=idx)+ time.sleep(0.3)++ # Clear any prior content+ driver.hotkey(safari_pid, ["cmd", "a"])+ time.sleep(0.1)++ driver.type_text_chars(safari_pid, "hello safari")+ time.sleep(0.6)++ after = driver.get_window_state(safari_pid, wid)+ assert after.has_text("hello safari"), (+ f"Typed text not visible in AX tree:\n{after.tree}"+ )++ def test_type_text_with_type_text_tool(self, driver, safari_pid, ux_guard):+ """Use type_text (not type_text_chars) and verify result."""+ wid = driver.find_window(safari_pid)++ before = driver.get_window_state(safari_pid, wid)+ idx = before.find_text_field(skip_url_bar=True)+ assert idx is not None++ driver.click(safari_pid, wid, element_index=idx)+ time.sleep(0.3)++ driver.hotkey(safari_pid, ["cmd", "a"])+ time.sleep(0.1)++ driver.type_text(safari_pid, "type_text test")+ time.sleep(0.6)++ after = driver.get_window_state(safari_pid, wid)+ assert after.has_text("type_text test"), (+ f"type_text result not in AX tree:\n{after.tree}"+ )+++class TestSafariCheckbox:+ """Toggle the checkbox."""++ def test_checkbox_toggle(self, driver, safari_pid, ux_guard):+ wid = driver.find_window(safari_pid)++ before = driver.get_window_state(safari_pid, wid, query="checkbox")+ idx = Tree.find(before.tree, role="AXCheckBox")+ if idx is None:+ # Fallback: find by label text+ idx = before.find_element("unchecked")+ assert idx is not None, f"Checkbox not found:\n{before.tree}"++ driver.click(safari_pid, wid, element_index=idx)+ time.sleep(0.4)++ after = driver.get_window_state(safari_pid, wid)+ assert after.has_text("checked"), (+ f"Checkbox did not toggle to checked:\n{after.tree}"+ )+++class TestSafariSelect:+ """Change the dropdown selection."""++ def test_select_option_b(self, driver, safari_pid, ux_guard):+ wid = driver.find_window(safari_pid)++ before = driver.get_window_state(safari_pid, wid, query="Option")+ idx = Tree.find(before.tree, role="AXPopUpButton")+ if idx is None:+ idx = Tree.find(before.tree, role="AXComboBox")+ if idx is None:+ idx = before.find_element("Option A")+ assert idx is not None, f"Select/dropdown not found:\n{before.tree}"++ driver.click(safari_pid, wid, element_index=idx)+ time.sleep(0.5)++ # Option B in the dropdown menu+ state2 = driver.get_window_state(safari_pid, wid)+ opt_b = Tree.find(state2.tree, label="Option B")+ if opt_b is not None:+ driver.click(safari_pid, wid, element_index=opt_b)+ else:+ # Fallback: set value directly+ driver.call_tool("set_value", {+ "pid": safari_pid, "window_id": wid,+ "element_index": idx, "value": "b",+ })+ time.sleep(0.4)++ after = driver.get_window_state(safari_pid, wid)+ assert after.has_text("Option B") or after.has_text("selected: Option B"), (+ f"Dropdown did not select Option B:\n{after.tree}"+ )+++class TestSafariTextarea:+ """Type into the textarea.++ Safari's WKWebView AXTextArea does not reliably set keyboard focus via+ AXPress in background mode. We use two strategies in sequence:+ 1. Click the text INPUT first to warm up Safari's web content focus engine.+ 2. Then click the textarea, wait 0.8s, type.+ """++ def test_textarea_input(self, driver, safari_pid, ux_guard):+ wid = driver.find_window(safari_pid)++ state = driver.get_window_state(safari_pid, wid)++ # Step 1: click the text input first to put focus inside Safari's web content+ tf_idx = state.find_text_field(skip_url_bar=True)+ if tf_idx is not None:+ driver.click(safari_pid, wid, element_index=tf_idx)+ time.sleep(0.3)++ # Step 2: click the textarea+ ta_idx = Tree.find(state.tree, role="AXTextArea")+ assert ta_idx is not None, f"Textarea not found:\n{state.tree[:500]}"+ driver.click(safari_pid, wid, element_index=ta_idx)+ time.sleep(0.8) # longer settle for WKWebView textarea focus++ # Step 3: type — keystrokes go to the focused element+ driver.call_tool("type_text_chars", {+ "pid": safari_pid,+ "text": "multi line",+ "delay_ms": 50,+ })+ time.sleep(0.6)++ after = driver.get_window_state(safari_pid, wid)+ val = Tree.ax_value(after.tree, ta_idx)+ print(f"\n textarea AX value: {val!r}")+ assert after.has_text("multi line"), (+ f"Textarea text not in AX tree; val={val!r};\n{after.tree[:800]}"+ )+++class TestSafariCanvas:+ """Send a pixel click to the canvas area without stealing focus.++ Background note: synthetic CGEvent pixel clicks to backgrounded browser+ windows may not trigger DOM mousedown events (web content is only rendered+ and interactive when the window is frontmost). This test therefore only+ asserts the key UX invariant: the click is delivered without error and+ without stealing focus. A DOM update (AX tree change or screenshot diff)+ is logged as a bonus if it occurs.+ """++ def test_canvas_pixel_click_no_focus_steal(self, driver, safari_pid, ux_guard):+ wid = driver.find_window(safari_pid)++ before = driver.get_window_state(safari_pid, wid)+ print(f"\n screenshot: {before.screenshot_width}x{before.screenshot_height}")++ from harness.cv import decode, diff_ratio+ before_img = decode(before.screenshot_b64)++ # Canvas is the last section on the page. With ~80px browser chrome+ # in a 906px window, the canvas element center is near y=88% of height.+ x = before.screenshot_width // 2+ y = int(before.screenshot_height * 0.88)+ print(f" pixel click at ({x}, {y})")++ result = driver.click(safari_pid, wid, x=x, y=y)+ time.sleep(0.5)++ after = driver.get_window_state(safari_pid, wid)++ # Bonus: check if DOM or screenshot updated (non-deterministic in background)+ canvas_clicked = "canvas: clicked" in after.tree+ after_img = decode(after.screenshot_b64)+ ratio = diff_ratio(before_img, after_img)+ print(f" canvas JS fired: {canvas_clicked}, diff_ratio: {ratio:.4f}")++ # Primary assertion: click succeeded (no exception was raised)+ # UX assertion: via ux_guard fixture teardown (no focus steal)+ assert result is not None, "click tool returned None"+++class TestSafariPressKey:+ """Press keys without stealing focus."""++ def test_tab_key(self, driver, safari_pid, ux_guard):+ driver.press_key(safari_pid, "tab")+ time.sleep(0.3)+ # No assertion on state — just verifying no exception + no UX violation++ def test_escape_key(self, driver, safari_pid, ux_guard):+ driver.press_key(safari_pid, "escape")+ time.sleep(0.2)diff --git a/libs/cua-driver-rs/tests/integration/v2/test_tauri.py b/libs/cua-driver-rs/tests/integration/v2/test_tauri.pynew file mode 100644index 0000000000..e1bb465f56--- /dev/null+++ b/libs/cua-driver-rs/tests/integration/v2/test_tauri.py@@ -0,0 +1,155 @@+"""Tauri desktop-test-app background automation tests — v2 harness.++The Tauri test app (v0.2.2) renders a 4-panel WKWebView UI and exposes+an HTTP API on port 6769 that logs all mouse/keyboard/clipboard events.++Primary assertion: AX tree state changes.+Cross-check: HTTP event log.++Run: python3 -m pytest test_tauri.py -v+"""++from __future__ import annotations++import subprocess+import sys+import time+import os++import pytest++try:+ import requests as _requests+ _REQUESTS_AVAILABLE = True+except ImportError:+ _REQUESTS_AVAILABLE = False++_HERE = os.path.dirname(os.path.abspath(__file__))+sys.path.insert(0, _HERE)+from harness.driver import Driver+from harness import tree as Tree++_TAURI_API = "http://localhost:6769"+++def _http_events(base_url: str = _TAURI_API) -> list[dict]:+ if not _REQUESTS_AVAILABLE:+ return []+ try:+ resp = _requests.get(f"{base_url}/events", timeout=2)+ return resp.json() if resp.ok else []+ except Exception:+ return []+++def _http_reset(base_url: str = _TAURI_API) -> None:+ if not _REQUESTS_AVAILABLE:+ return+ try:+ _requests.post(f"{base_url}/reset", timeout=2)+ except Exception:+ pass++[email protected](autouse=True)+def _reactivate_focus(focus_monitor):+ _, pid = focus_monitor+ subprocess.run(+ ["osascript", "-e",+ f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'],+ check=False,+ )+ time.sleep(0.4)+++# ── tests ─────────────────────────────────────────────────────────────────────++class TestTauriButton:+ def test_button_click(self, driver, tauri_app, ux_guard):+ _, pid, base_url = tauri_app+ if pid == 0:+ pytest.skip("Tauri app pid not found")++ _http_reset(base_url)+ wid = driver.find_window(pid)++ before = driver.get_window_state(pid, wid, query="button")+ idx = before.find_element("click")+ if idx is None:+ idx = Tree.find(before.tree, role="AXButton")+ assert idx is not None, f"Button not found:\n{before.tree}"++ driver.click(pid, wid, element_index=idx)+ time.sleep(0.5)++ after = driver.get_window_state(pid, wid)+ # Verify some state change happened (counter or AX value)+ changed = after.tree != before.tree+ print(f"\n tauri tree changed: {changed}")++ # Cross-check: HTTP events should contain a mouse event+ events = _http_events(base_url)+ print(f" tauri events: {len(events)}")+ # Either AX tree changed or HTTP logged a mouse event+ assert changed or any(+ e.get("type") in ("mouse_down", "click", "mousedown") for e in events+ ), f"No observable change after button click:\n{after.tree}"+++class TestTauriTextInput:+ def test_type_text(self, driver, tauri_app, ux_guard):+ _, pid, base_url = tauri_app+ if pid == 0:+ pytest.skip("Tauri app pid not found")++ _http_reset(base_url)+ wid = driver.find_window(pid)++ before = driver.get_window_state(pid, wid)+ idx = before.find_text_field(skip_url_bar=False)+ if idx is None:+ idx = Tree.find(before.tree, role="AXTextField")+ if idx is None:+ idx = Tree.find(before.tree, role="AXTextArea")+ assert idx is not None, f"Text input not found:\n{before.tree}"++ driver.click(pid, wid, element_index=idx)+ time.sleep(0.3)+ driver.hotkey(pid, ["cmd", "a"])+ time.sleep(0.1)+ driver.type_text_chars(pid, "tauri test")+ time.sleep(0.6)++ after = driver.get_window_state(pid, wid)+ text_in_tree = after.has_text("tauri test")+ print(f"\n text in AX tree: {text_in_tree}")++ events = _http_events(base_url)+ key_events = [e for e in events if e.get("type") in ("key_down", "keydown", "input")]+ print(f" key events: {len(key_events)}")++ assert text_in_tree or len(key_events) >= len("tauri test"), (+ f"Text not found in AX tree and insufficient key events:\n{after.tree}"+ )+++class TestTauriNoFocusSteal:+ def test_multiple_actions_no_focus_steal(self, driver, tauri_app, ux_guard):+ _, pid, base_url = tauri_app+ if pid == 0:+ pytest.skip("Tauri app pid not found")++ wid = driver.find_window(pid)++ # Take 3 actions in sequence — UX guard checks no focus steal+ before = driver.get_window_state(pid, wid)+ idx = Tree.find(before.tree, role="AXButton")+ if idx is not None:+ driver.click(pid, wid, element_index=idx)+ time.sleep(0.3)++ driver.press_key(pid, "tab")+ time.sleep(0.2)+ driver.press_key(pid, "tab")+ time.sleep(0.2)+ # ux_guard teardown will assert_clean()diff --git a/libs/cua-driver/scripts/install.sh b/libs/cua-driver/scripts/install.shindex 0cf50eb8d2..180a2c05da 100755--- a/libs/cua-driver/scripts/install.sh+++ b/libs/cua-driver/scripts/install.sh@@ -68,8 +68,14 @@ if [[ -n "${CUA_DRIVER_VERSION:-}" ]]; thenlog "using version from CUA_DRIVER_VERSION: $TAG"elselog "resolving latest $TAG_PREFIX* release via GitHub API"+ # `grep -v cua-driver-rs` is defense-in-depth — the Rust port (cua-driver-rs)+ # publishes to the same repo under tag prefix `cua-driver-rs-v*` and the+ # current grep regex `cua-driver-v[^"]+` already excludes it (differs at+ # position 11: 'r' vs 'v'), but the negation guards against any future+ # regex tweak that might accidentally widen the match.TAG=$(curl -fsSL "https://api.github.com/repos/$REPO/releases?per_page=40" \| grep -Eo '"tag_name":[[:space:]]*"'"${TAG_PREFIX}"'[^"]+"' \+ | grep -v 'cua-driver-rs' \| sed -E 's/.*"'"${TAG_PREFIX}"'([0-9]+[.][0-9]+[.][0-9]+)"/\1/' \| sort -t. -k1,1nr -k2,2nr -k3,3nr \| head -n 1 \