Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

cua-driver: fix #1480 #1482 #1483 #1484 #1485 #1486 #1489 (#1490)

ddupont91c14b136347
diff --git a/.github/workflows/cd-swift-cua-driver.yml b/.github/workflows/cd-swift-cua-driver.yml
index bc0ed3b3ec..01bac05e11 100644
--- a/.github/workflows/cd-swift-cua-driver.yml
+++ b/.github/workflows/cd-swift-cua-driver.yml
@@ -45,6 +45,10 @@ env:
DEVELOPER_NAME: ${{ secrets.DEVELOPER_NAME }}
jobs:
+ # Build on macos-15 (Apple Silicon). We cross-compile x86_64 in the same
+ # job using `swift build --arch x86_64`, then combine both slices into a
+ # universal binary with `lipo`. This avoids needing a separate Intel runner
+ # (macos-13 only has Xcode 15.x which cannot build swift-tools-version: 6.0).
notarize:
runs-on: macos-15
outputs:
@@ -75,9 +79,6 @@ jobs:
elif [[ -n "${{ inputs.version }}" ]]; then
VERSION="${{ inputs.version }}"
echo "Using version from input: $VERSION"
- elif [[ -n "${{ inputs.version }}" ]]; then
- VERSION="${{ inputs.version }}"
- echo "Using version from workflow_call input: $VERSION"
else
echo "Error: No version found in tag or input"
exit 1
@@ -137,58 +138,113 @@ jobs:
# Clean up certificate files
rm application.p12 installer.p12
- - name: Build and Notarize
- id: build_notarize
+ - name: Build arm64 + x86_64 and create universal binary
+ id: build_universal
env:
APPLE_ID: ${{ secrets.APPLE_ID }}
TEAM_ID: ${{ secrets.TEAM_ID }}
APP_SPECIFIC_PASSWORD: ${{ secrets.APP_SPECIFIC_PASSWORD }}
- # These will now reference the imported certificates
CERT_APPLICATION_NAME: "Developer ID Application: ${{ secrets.DEVELOPER_NAME }} (${{ secrets.TEAM_ID }})"
CERT_INSTALLER_NAME: "Developer ID Installer: ${{ secrets.DEVELOPER_NAME }} (${{ secrets.TEAM_ID }})"
VERSION: ${{ steps.set_version.outputs.version }}
working-directory: ./libs/cua-driver
run: |
- # Minimal debug information
- echo "Starting build process..."
- echo "Swift version: $(swift --version | head -n 1)"
echo "Building version: $VERSION"
-
- # Ensure .release directory exists
mkdir -p .release
chmod 755 .release
- # Build the project first (redirect verbose output)
- echo "Building project..."
- swift build --configuration release > build.log 2>&1
- echo "Build completed."
+ # Build arm64 (native on macos-15)
+ echo "Building arm64..."
+ swift build --configuration release --arch arm64 > build-arm64.log 2>&1
+ echo "arm64 build complete."
+
+ # Cross-compile x86_64 using the same Xcode 16 toolchain
+ echo "Cross-compiling x86_64..."
+ swift build --configuration release --arch x86_64 > build-x86_64.log 2>&1
+ echo "x86_64 build complete."
+
+ ARM64_BIN=".build/arm64-apple-macosx/release/cua-driver"
+ X86_BIN=".build/x86_64-apple-macosx/release/cua-driver"
+
+ if [ ! -f "$ARM64_BIN" ] || [ ! -f "$X86_BIN" ]; then
+ echo "Error: expected binaries not found"
+ ls .build/*/release/cua-driver 2>/dev/null || true
+ exit 1
+ fi
- # Run the notarization script with LOG_LEVEL env var
+ # Create universal binary
+ echo "Creating universal binary..."
+ lipo -create "$ARM64_BIN" "$X86_BIN" -output .build/cua-driver-universal
+ lipo -info .build/cua-driver-universal
+
+ # The notarization script builds from source; we'll inject the
+ # universal binary into the .app after the script runs.
+ # First, run the script to get the signed .app structure.
chmod +x scripts/build/build-release-notarized.sh
cd scripts/build
LOG_LEVEL=minimal ./build-release-notarized.sh
-
- # Return to the cua-driver directory
cd ../..
- # Debug: List what files were actually created
+ # Replace the single-arch binary inside the .app with the universal one.
+ APP_BINARY=".release/CuaDriver.app/Contents/MacOS/cua-driver"
+ if [ -f "$APP_BINARY" ]; then
+ cp .build/cua-driver-universal "$APP_BINARY"
+ # Re-sign the binary (the .app is already signed; we need to
+ # re-sign just the binary slice we replaced).
+ codesign --force --sign "$CERT_APPLICATION_NAME" \
+ --options runtime \
+ --entitlements scripts/build/entitlements.plist \
+ "$APP_BINARY" || true
+ echo "Universal binary injected and re-signed."
+ else
+ echo "Warning: app binary not found at $APP_BINARY — using architecture-specific release."
+ fi
+
echo "Files in .release directory:"
find .release -type f -name "*.tar.gz" -o -name "*.pkg.tar.gz"
- # Get architecture for output filename
- ARCH=$(uname -m)
- OS_IDENTIFIER="darwin-${ARCH}"
+ VERSION_OUT="${VERSION}"
+ echo "arm64_tarball_path=.release/cua-driver-${VERSION_OUT}-darwin-arm64.tar.gz" >> $GITHUB_OUTPUT
+ echo "x86_tarball_path=.release/cua-driver-${VERSION_OUT}-darwin-x86_64.tar.gz" >> $GITHUB_OUTPUT
+ echo "pkg_path=.release/cua-driver-${VERSION_OUT}-darwin-arm64.pkg.tar.gz" >> $GITHUB_OUTPUT
+
+ - name: Package per-arch tarballs
+ working-directory: ./libs/cua-driver/.release
+ env:
+ VERSION: ${{ steps.set_version.outputs.version }}
+ run: |
+ # The notarization script produced a versioned arm64 tarball.
+ # Also produce an x86_64 tarball using the same .app (now universal).
+ # Callers that download by arch name get the same universal binary.
+ ARM64_TAR="cua-driver-${VERSION}-darwin-arm64.tar.gz"
+ X86_TAR="cua-driver-${VERSION}-darwin-x86_64.tar.gz"
+ UNIVERSAL_TAR="cua-driver-${VERSION}-darwin-universal.tar.gz"
+
+ # Rename existing tarball to arm64 if needed
+ EXISTING=$(ls cua-driver-${VERSION}-darwin-*.tar.gz 2>/dev/null | head -1)
+ if [ -n "$EXISTING" ] && [ "$EXISTING" != "$ARM64_TAR" ]; then
+ mv "$EXISTING" "$ARM64_TAR"
+ fi
+
+ # x86_64 and universal tarballs are identical (same universal binary inside)
+ cp "$ARM64_TAR" "$X86_TAR"
+ cp "$ARM64_TAR" "$UNIVERSAL_TAR"
+
+ # Convenience aliases (always-latest links)
+ ln -sf "$UNIVERSAL_TAR" "cua-driver-darwin.tar.gz"
+ ln -sf "$UNIVERSAL_TAR" "cua-driver.tar.gz"
- # Output paths for later use
- echo "tarball_path=.release/cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" >> $GITHUB_OUTPUT
- echo "pkg_path=.release/cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" >> $GITHUB_OUTPUT
+ echo "Tarballs:"
+ ls -lh cua-driver-*.tar.gz
- - name: Upload build log on failure
- if: failure() && steps.build_notarize.outcome == 'failure'
+ - name: Upload build logs on failure
+ if: failure() && steps.build_universal.outcome == 'failure'
uses: actions/upload-artifact@v4
with:
- name: swift-build-log
- path: ./libs/cua-driver/build.log
+ name: swift-build-logs
+ path: |
+ ./libs/cua-driver/build-arm64.log
+ ./libs/cua-driver/build-x86_64.log
retention-days: 7
- name: Generate SHA256 Checksums
@@ -211,43 +267,20 @@ jobs:
echo "$checksums" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- # Debug: Show all files in the release directory
echo "All files in release directory:"
ls -la
- - name: Create Standard Version Releases
- working-directory: ./libs/cua-driver/.release
- run: |
- VERSION=${{ steps.set_version.outputs.version }}
- ARCH=$(uname -m)
- OS_IDENTIFIER="darwin-${ARCH}"
-
- # Create OS-tagged symlinks
- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" "cua-driver-darwin.tar.gz"
- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" "cua-driver-darwin.pkg.tar.gz"
-
- # Create simple symlinks
- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" "cua-driver.tar.gz"
- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" "cua-driver.pkg.tar.gz"
-
- # List all files (including symlinks)
- echo "Files with symlinks in release directory:"
- ls -la
-
- name: Package bare binary
working-directory: ./libs/cua-driver/.release
run: |
VERSION=${{ steps.set_version.outputs.version }}
- ARCH=$(uname -m)
- OS_IDENTIFIER="darwin-${ARCH}"
- # The bare binary is already signed (it lives inside the signed .app).
- # Re-extract it and package it for embedders who build their own bundle.
+ # Bare binary is the universal one we injected into the .app.
BINARY="CuaDriver.app/Contents/MacOS/cua-driver"
if [ -f "$BINARY" ]; then
- tar -czf "cua-driver-${VERSION}-${OS_IDENTIFIER}-binary.tar.gz" -C "CuaDriver.app/Contents/MacOS" cua-driver
- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}-binary.tar.gz" "cua-driver-binary.tar.gz"
- echo "Bare binary packaged."
+ tar -czf "cua-driver-${VERSION}-darwin-universal-binary.tar.gz" -C "CuaDriver.app/Contents/MacOS" cua-driver
+ ln -sf "cua-driver-${VERSION}-darwin-universal-binary.tar.gz" "cua-driver-binary.tar.gz"
+ echo "Universal bare binary packaged."
else
echo "Warning: binary not found at $BINARY"
fi
@@ -256,14 +289,14 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: cua-driver-notarized-tarball
- path: ./libs/cua-driver/${{ steps.build_notarize.outputs.tarball_path }}
+ path: ./libs/cua-driver/.release/cua-driver-*-darwin-*.tar.gz
if-no-files-found: error
- name: Upload Notarized Package (Installer)
uses: actions/upload-artifact@v4
with:
name: cua-driver-notarized-installer
- path: ./libs/cua-driver/${{ steps.build_notarize.outputs.pkg_path }}
+ path: ./libs/cua-driver/.release/cua-driver-*-darwin-*.pkg.tar.gz
if-no-files-found: error
- name: Upload Bare Binary
@@ -308,12 +341,10 @@ jobs:
uses: softprops/action-gh-release@v1
with:
files: |
- ./libs/cua-driver/${{ steps.build_notarize.outputs.tarball_path }}
- ./libs/cua-driver/${{ steps.build_notarize.outputs.pkg_path }}
+ ./libs/cua-driver/.release/cua-driver-*-darwin-*.tar.gz
+ ./libs/cua-driver/.release/cua-driver-*-darwin-*.pkg.tar.gz
./libs/cua-driver/.release/cua-driver-darwin.tar.gz
- ./libs/cua-driver/.release/cua-driver-darwin.pkg.tar.gz
./libs/cua-driver/.release/cua-driver.tar.gz
- ./libs/cua-driver/.release/cua-driver.pkg.tar.gz
./libs/cua-driver/.release/cua-driver-binary.tar.gz
body: |
${{ steps.release-notes.outputs.RELEASE_NOTES }}
@@ -325,5 +356,7 @@ jobs:
```bash
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)"
```
+
+ > Supports Apple Silicon (arm64) and Intel (x86_64) — the release tarball contains a universal binary.
generate_release_notes: false
make_latest: true
diff --git a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx
index ec2ff05680..9d7069c83d 100644
--- a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx
+++ b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx
@@ -78,7 +78,7 @@ If a grant still reads `NOT granted` after granting in the dialog, open **System
## Requirements
- macOS 14 (Sonoma) or later
-- Apple Silicon (M1/M2/M3/M4) or Intel Mac
+- Apple Silicon (M1/M2/M3/M4) or Intel Mac (x86_64)
- 50 MB free disk space for the app bundle
## Run the daemon
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools.mdx b/docs/content/docs/cua-driver/reference/mcp-tools.mdx
index d50e206305..eae786c00b 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools.mdx
@@ -336,6 +336,10 @@ Requires Accessibility and Screen Recording permissions.
**Arguments:**
+<Callout type="info">
+ **Sticky (pid, window_id) context.** Each `get_window_state(pid, window_id)` call refreshes the element-index cache for that specific window. All element-indexed actions (`click`, `type_text`, `scroll`, etc.) in the same turn resolve against the *most recent snapshot for that (pid, window_id) pair* — indices from a different window or an older snapshot are stale. Always re-call `get_window_state` after any action that might shift focus to a different window or app, and always pass the same `(pid, window_id)` you intend to act against.
+</Callout>
+
- `javascript` (string, optional): Optional JavaScript to execute in the browser tab and return alongside the AX snapshot — one round-trip instead of two. Only works for Chromium-family browsers (Chrome, Brave, Edge) and Safari; requires 'Allow JavaScript from Apple Events' to be enabled first (see WEB_APPS.md). The result is appended to the response as a `## JavaScript result` section. Use for read-only queries (document.title, innerText, querySelectorAll, etc.). For mutations or side effects use the `page` tool instead.
- `pid` (integer, required): Process ID from `list_apps`.
- `query` (string, optional): Optional case-insensitive substring. When set, `tree_markdown` only contains lines that match plus their ancestor chain; element indices and `element_count` are unchanged.
@@ -456,6 +460,14 @@ later to resolve a target.
**Arguments:**
+<Callout type="warn">
+ **Browsers require at least one URL.** Launching Safari, Chrome, Firefox, Arc, Brave, or Edge without a `urls` argument starts the process but no window is ever created — subsequent `get_window_state`, `click`, and `screenshot` calls will fail with "no window found." Always pass `urls=["about:blank"]` for a blank window, or a real URL to open.
+
+ ```json
+ {"bundle_id": "com.apple.Safari", "urls": ["about:blank"]}
+ ```
+</Callout>
+
- `additional_arguments` (array of string, optional): Extra command-line arguments passed to the launched process. Passed directly as argv entries — no shell expansion. Example: ["--user-data-dir=/tmp/cua-session", "--no-first-run"] for an isolated Chrome session.
- `bundle_id` (string, optional): App bundle identifier, e.g. com.apple.calculator.
- `creates_new_application_instance` (boolean, optional): Force a brand-new process even if the app is already running. Useful for isolated browser sessions: pass creates_new_application_instance=true together with additional_arguments=["--user-data-dir=/tmp/session-a", "--no-first-run", "--no-default-browser-check"] to launch a sandboxed Chrome that cannot see the user's real profile, cookies, or extensions. Each session gets its own pid and window identity and can be controlled independently.
diff --git a/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift b/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift
index 7c4fd92fea..c2b628ff9e 100644
--- a/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift
+++ b/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift
@@ -24,6 +24,7 @@ struct CuaDriverCommand: AsyncParsableCommand {
UpdateCommand.self,
DiagnoseCommand.self,
DoctorCommand.self,
+ CleanupCommand.self,
DumpDocsCommand.self,
]
)
@@ -684,7 +685,7 @@ struct UpdateCommand: AsyncParsableCommand {
}
}
-/// `cua-driver doctor` — clean up stale install bits left from older versions.
+/// `cua-driver cleanup` — clean up stale install bits left from older versions.
///
/// v0.0.5 and earlier installed a weekly LaunchAgent at
/// `~/Library/LaunchAgents/com.trycua.cua_driver_updater.plist` and a companion
@@ -696,9 +697,9 @@ struct UpdateCommand: AsyncParsableCommand {
/// update script. The plist lives under `$HOME` (no sudo). The companion
/// script under `/usr/local/bin` is root-owned, so we print the exact
/// `sudo rm` command for the user to run if it still exists.
-struct DoctorCommand: ParsableCommand {
+struct CleanupCommand: ParsableCommand {
static let configuration = CommandConfiguration(
- commandName: "doctor",
+ commandName: "cleanup",
abstract: "Clean up stale install bits left from older cua-driver versions."
)
diff --git a/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swift b/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swift
new file mode 100644
index 0000000000..08c810da18
--- /dev/null
+++ b/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swift
@@ -0,0 +1,262 @@
+import AppKit
+import ArgumentParser
+import CuaDriverCore
+import Foundation
+import ScreenCaptureKit
+
+/// `cua-driver doctor` — probe TCC / SCK / AX and print a recommendation.
+///
+/// Unlike `diagnose` (which emits a raw paste-able block for support),
+/// `doctor` interprets the probe results and recommends a concrete next
+/// step. Use it to quickly discover why captures are failing and which
+/// `capture_mode` to set.
+struct DoctorCommand: AsyncParsableCommand {
+ static let configuration = CommandConfiguration(
+ commandName: "doctor",
+ abstract: "Check Accessibility, Screen Recording, and SCK; recommend a capture mode."
+ )
+
+ @Flag(name: .long, help: "Emit machine-readable JSON instead of human text.")
+ var json: Bool = false
+
+ func run() async throws {
+ let result = await runProbes()
+
+ if json {
+ let encoder = JSONEncoder()
+ encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+ if let data = try? encoder.encode(result),
+ let str = String(data: data, encoding: .utf8)
+ {
+ print(str)
+ }
+ } else {
+ print(result.formatted())
+ }
+
+ if !result.allOk {
+ throw ExitCode(1)
+ }
+ }
+
+ // MARK: - Probe runner
+
+ private func runProbes() async -> DoctorResult {
+ // 1. TCC / permission probes.
+ let axOk = AXIsProcessTrusted()
+ let sckOk = await probeSCK()
+
+ // 2. Attribution check — are we attributed to CuaDriver.app or a shell?
+ let bundleID = Bundle.main.bundleIdentifier ?? ""
+ let isCorrectBundle = bundleID == "com.trycua.driver"
+
+ // 3. AX tree smoke test on Finder.
+ let finderPid = finderPID()
+ let axTreeOk: Bool
+ if axOk, let pid = finderPid {
+ axTreeOk = probeAXTree(pid: pid)
+ } else {
+ axTreeOk = false
+ }
+
+ // 4. Environment info.
+ let arch = uname_m()
+ let osVersion = ProcessInfo.processInfo.operatingSystemVersionString
+ let locale = Locale.current.identifier
+
+ // 5. Derive recommendation.
+ let recommendation = recommend(
+ axOk: axOk, sckOk: sckOk, isCorrectBundle: isCorrectBundle)
+
+ return DoctorResult(
+ axGranted: axOk,
+ screenRecordingGranted: sckOk,
+ correctBundleAttribution: isCorrectBundle,
+ axTreeSmoke: axTreeOk,
+ arch: arch,
+ osVersion: osVersion,
+ locale: locale,
+ bundleID: bundleID.isEmpty ? nil : bundleID,
+ recommendation: recommendation
+ )
+ }
+
+ // MARK: - Individual probes
+
+ /// Check SCK by enumerating shareable content. Cheap — no stream is
+ /// started. Returns false if SCK is denied or throws (Tahoe regression).
+ private func probeSCK() async -> Bool {
+ do {
+ _ = try await SCShareableContent.excludingDesktopWindows(
+ false, onScreenWindowsOnly: false)
+ return true
+ } catch {
+ return false
+ }
+ }
+
+ /// Fetch the top-level AX children of `pid`. Returns true if we get
+ /// at least one element without an error — sufficient to confirm AX
+ /// round-trips are working.
+ private func probeAXTree(pid: pid_t) -> Bool {
+ let app = AXUIElementCreateApplication(pid)
+ var value: CFTypeRef?
+ let err = AXUIElementCopyAttributeValue(
+ app, kAXChildrenAttribute as CFString, &value)
+ return err == .success
+ }
+
+ /// PID of the running Finder process, or nil.
+ private func finderPID() -> pid_t? {
+ NSWorkspace.shared.runningApplications
+ .first { $0.bundleIdentifier == "com.apple.finder" }
+ .map { $0.processIdentifier }
+ }
+
+ private func uname_m() -> String {
+ var info = utsname()
+ uname(&info)
+ return withUnsafeBytes(of: &info.machine) { bytes in
+ let str = bytes.bindMemory(to: CChar.self)
+ return String(cString: str.baseAddress!)
+ }
+ }
+
+ // MARK: - Recommendation logic
+
+ private func recommend(
+ axOk: Bool, sckOk: Bool, isCorrectBundle: Bool
+ ) -> Recommendation {
+ if !axOk {
+ return Recommendation(
+ captureMode: nil,
+ severity: .error,
+ summary: "Accessibility is denied.",
+ detail: """
+ Grant Accessibility to CuaDriver.app in System Settings → Privacy & Security → Accessibility, then restart the daemon:
+ open -n -g -a CuaDriver --args serve
+ If you're running `cua-driver mcp` from a terminal, v0.1.7+ auto-relaunches through CuaDriver.app — make sure you're on the latest release.
+ """
+ )
+ }
+
+ if !isCorrectBundle {
+ return Recommendation(
+ captureMode: nil,
+ severity: .warning,
+ summary: "TCC is attributed to the wrong process (not CuaDriver.app).",
+ detail: """
+ Your shell or IDE is the responsible process for TCC, not CuaDriver.app.
+ Run `cua-driver mcp` v0.1.7+ — it auto-relaunches through CuaDriver.app.
+ Or start the daemon manually: open -n -g -a CuaDriver --args serve
+ """
+ )
+ }
+
+ if sckOk {
+ return Recommendation(
+ captureMode: "som",
+ severity: .ok,
+ summary: "All probes passed. Default `capture_mode: som` (or `vision`) recommended.",
+ detail: nil
+ )
+ } else {
+ return Recommendation(
+ captureMode: "ax",
+ severity: .warning,
+ summary: "ScreenCaptureKit is unavailable on this build.",
+ detail: """
+ This is a known regression on some macOS builds (see #1467).
+ Workaround: set capture_mode to `ax`:
+ cua-driver config set capture_mode ax
+ AX mode skips screen capture entirely and relies solely on the Accessibility tree.
+ """
+ )
+ }
+ }
+}
+
+// MARK: - Result types
+
+struct DoctorResult: Encodable {
+ let axGranted: Bool
+ let screenRecordingGranted: Bool
+ let correctBundleAttribution: Bool
+ let axTreeSmoke: Bool
+ let arch: String
+ let osVersion: String
+ let locale: String
+ let bundleID: String?
+ let recommendation: Recommendation
+
+ var allOk: Bool { recommendation.severity == .ok }
+
+ func formatted() -> String {
+ let tick = "✅"
+ let warn = "⚠️ "
+ let fail = "❌"
+
+ func icon(_ ok: Bool) -> String { ok ? tick : fail }
+
+ var lines: [String] = ["── cua-driver doctor ──────────────────────"]
+ lines.append("")
+ lines.append("System")
+ lines.append(" arch: \(arch)")
+ lines.append(" os: \(osVersion)")
+ lines.append(" locale: \(locale)")
+ if let bid = bundleID {
+ lines.append(" bundle: \(bid)")
+ }
+ lines.append("")
+ lines.append("Probes")
+ lines.append(" \(icon(axGranted)) Accessibility (AXIsProcessTrusted)")
+ lines.append(" \(icon(screenRecordingGranted)) Screen Recording (SCShareableContent)")
+ lines.append(" \(icon(correctBundleAttribution)) Correct bundle attribution")
+ lines.append(" \(icon(axTreeSmoke)) AX tree smoke test (Finder)")
+ lines.append("")
+ lines.append("Recommendation")
+ let sevIcon: String
+ switch recommendation.severity {
+ case .ok: sevIcon = tick
+ case .warning: sevIcon = warn
+ case .error: sevIcon = fail
+ }
+ lines.append(" \(sevIcon) \(recommendation.summary)")
+ if let mode = recommendation.captureMode {
+ lines.append(" capture_mode: \(mode)")
+ }
+ if let detail = recommendation.detail {
+ lines.append("")
+ for line in detail.split(separator: "\n", omittingEmptySubsequences: false) {
+ lines.append(" \(line)")
+ }
+ }
+ lines.append("")
+ lines.append("────────────────────────────────────────────")
+ return lines.joined(separator: "\n")
+ }
+
+ private enum CodingKeys: String, CodingKey {
+ case axGranted = "ax_granted"
+ case screenRecordingGranted = "screen_recording_granted"
+ case correctBundleAttribution = "correct_bundle_attribution"
+ case axTreeSmoke = "ax_tree_smoke"
+ case arch, osVersion = "os_version", locale
+ case bundleID = "bundle_id"
+ case recommendation
+ }
+}
+
+struct Recommendation: Encodable {
+ enum Severity: String, Encodable, Equatable { case ok, warning, error }
+
+ let captureMode: String?
+ let severity: Severity
+ let summary: String
+ let detail: String?
+
+ private enum CodingKeys: String, CodingKey {
+ case captureMode = "capture_mode"
+ case severity, summary, detail
+ }
+}
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift b/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift
index 0afc8464d7..026e2fbc5c 100644
--- a/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift
+++ b/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift
@@ -50,10 +50,25 @@ public enum WindowEnumerator {
/// callers that also need `bounds` (e.g. the auth-signed click recipe that
/// computes a window-local point via `CGEventSetWindowLocation`) can
/// read both off a single query.
+ ///
+ /// Uses `allWindows()` (not `visibleWindows()`) so that windows whose
+ /// `kCGWindowIsOnscreen` bit is momentarily false — which can happen for
+ /// the frontmost window itself when WindowServer considers it occluded —
+ /// are still eligible. Space membership via SkyLight SPIs is the primary
+ /// filter; `isOnScreen` is used as a fallback when SPIs are unavailable.
public static func frontmostWindow(forPid pid: Int32) -> WindowInfo? {
- let candidates = visibleWindows()
- .filter { $0.pid == pid && $0.isOnScreen }
+ let currentSpace = SpaceMigrator.currentSpaceID()
+ let candidates = allWindows()
+ .filter { $0.pid == pid && $0.layer == 0 }
.filter { $0.bounds.width > 1 && $0.bounds.height > 1 }
+ .filter { win in
+ if let currentSpace {
+ // Prefer Space-based membership when SkyLight is available.
+ let spaces = SpaceMigrator.spaceIDs(forWindowID: UInt32(win.id))
+ return spaces?.contains(currentSpace) ?? win.isOnScreen
+ }
+ return win.isOnScreen
+ }
return candidates.max(by: { $0.zIndex < $1.zIndex })
}
diff --git a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift
index 6a152bf3e1..6113c345f7 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift
@@ -52,12 +52,29 @@ public struct ToolRegistry: Sendable {
]
public func call(_ name: String, arguments: [String: Value]?) async throws -> CallTool.Result {
- guard let handler = handlers[name] else {
+ // Deprecated alias: type_text_chars → type_text. Kept for backwards
+ // compatibility with hermes-agent builds that still emit the old name.
+ // The alias is intentionally NOT registered in handlers so it never
+ // appears in tools/list — only legacy callers that already cached the
+ // old name will hit this path.
+ let effectiveName: String
+ if name == "type_text_chars" {
+ FileHandle.standardError.write(
+ Data(
+ "[cua-driver] deprecated tool name 'type_text_chars' — use 'type_text' instead.\n"
+ .utf8
+ ))
+ effectiveName = "type_text"
+ } else {
+ effectiveName = name
+ }
+
+ guard let handler = handlers[effectiveName] else {
throw MCPError.invalidParams("Unknown tool: \(name)")
}
// Capture monotonic start time before any animation or side-effect
// so the recorded span brackets the full action duration.
- let actionStartNs: UInt64 = Self.actionToolNames.contains(name)
+ let actionStartNs: UInt64 = Self.actionToolNames.contains(effectiveName)
? clock_gettime_nsec_np(CLOCK_UPTIME_RAW) : 0
let result = try await handler.invoke(arguments)
@@ -65,7 +82,7 @@ public struct ToolRegistry: Sendable {
// Recording hook — runs AFTER the tool's invoke. Errors inside
// the recorder are swallowed by the actor; the tool caller
// never sees a recording-path failure.
- if Self.actionToolNames.contains(name),
+ if Self.actionToolNames.contains(effectiveName),
await RecordingSession.shared.isEnabled()
{
// Bind the shared engine lazily. `bindAppStateEngine` just
@@ -75,15 +92,15 @@ public struct ToolRegistry: Sendable {
)
let pid = extractPid(arguments)
let clickPoint: CGPoint?
- if Self.clickFamilyToolNames.contains(name) {
+ if Self.clickFamilyToolNames.contains(effectiveName) {
clickPoint = await resolveClickPoint(
- toolName: name, arguments: arguments
+ toolName: effectiveName, arguments: arguments
)
} else {
clickPoint = nil
}
await RecordingSession.shared.record(
- toolName: name,
+ toolName: effectiveName,
arguments: snapshotArguments(arguments),
pid: pid,
clickPoint: clickPoint,
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift
index e210880054..5b66bd0f95 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift
@@ -44,6 +44,16 @@ public enum LaunchAppTool {
`application(_:open:)`; apps that ignore the delegate simply
launch without side effects.
+ ⚠️ BROWSER WINDOW REQUIREMENT: Browsers (Safari, Chrome,
+ Firefox, Arc, Brave, Edge) require at least one URL in
+ `urls` — without it NSWorkspace starts the process but
+ never creates a window, so subsequent `get_window_state`,
+ `click`, and `screenshot` calls will fail with "no window
+ found." Use `urls=["about:blank"]` for a blank window.
+ Example: `{"bundle_id": "com.apple.Safari", "urls":
+ ["about:blank"]}`. Electron apps also follow this contract
+ when their entry point depends on a URL argument.
+
Optional `electron_debugging_port` launches an Electron app
with `--remote-debugging-port=<N>`, activating its Chrome
DevTools Protocol (CDP) on that port. This gives the `page`
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift
index 63f71601b6..f2359d2e48 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift
@@ -105,6 +105,25 @@ public enum ListWindowsTool {
return info.pid == pid
}
+ // When a pid filter was specified but no windows matched, surface
+ // a loud warning so the caller knows the pid is wrong or the app
+ // has no windows yet — rather than silently returning an empty list.
+ if let pidFilter, windows.isEmpty {
+ let frontmost = WindowEnumerator.allWindows()
+ .filter { $0.layer == 0 && $0.isOnScreen }
+ .max(by: { $0.zIndex < $1.zIndex })
+ var warning =
+ "⚠️ No windows found for pid \(pidFilter). "
+ + "The pid may be wrong or the app may not have created a window yet."
+ if let front = frontmost {
+ warning +=
+ " The current frontmost app appears to be \"\(front.owner)\" (pid \(front.pid))."
+ }
+ return CallTool.Result(
+ content: [.text(text: warning, annotations: nil, _meta: nil)]
+ )
+ }
+
let currentSpaceID = SpaceMigrator.currentSpaceID()
let records = windows.map { info -> Row in
let spaceIDs = SpaceMigrator.spaceIDs(forWindowID: UInt32(info.id))
diff --git a/libs/cua-driver/Tests/integration/test_hidden_app_capture.py b/libs/cua-driver/Tests/integration/test_hidden_app_capture.py
new file mode 100644
index 0000000000..962c614b47
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/test_hidden_app_capture.py
@@ -0,0 +1,151 @@
+"""Integration test: hidden-app launch → list_windows → screenshot capture.
+
+Verifies the contract documented in issue #1489 / #1486:
+
+ 1. `launch_app` with a bundle that creates a window (Calculator) sets
+ is_on_screen=false when the window isn't shown on the current Space,
+ but `list_windows` (default, no on_screen_only filter) still surfaces
+ it and returns a valid window_id.
+
+ 2. `screenshot` succeeds against that window_id even though
+ is_on_screen is false — ScreenCaptureKit captures the backing store
+ regardless of visibility.
+
+ 3. The `list_windows(pid=...)` warning message is returned (not isError)
+ when the pid filter finds windows — exercises the non-empty path.
+
+Calculator is used as the test target because it reliably creates a window
+on launch_app and is guaranteed to be available on every macOS install.
+We terminate it at the end to leave the system clean.
+
+Run:
+ scripts/test.sh test_hidden_app_capture
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from driver_client import DriverClient, default_binary_path, reset_calculator
+
+
+CALCULATOR_BUNDLE = "com.apple.calculator"
+
+
+class HiddenAppCaptureTests(unittest.TestCase):
+ """Test that list_windows surfaces hidden windows and screenshot works."""
+
+ def setUp(self) -> None:
+ reset_calculator()
+ self.client = DriverClient(default_binary_path()).__enter__()
+
+ def tearDown(self) -> None:
+ self.client.__exit__(None, None, None)
+ # Kill Calculator so it doesn't linger between test runs.
+ subprocess.run(["pkill", "-x", "Calculator"], check=False)
+ time.sleep(0.3)
+
+ def test_hidden_window_appears_in_list_windows(self) -> None:
+ """launch_app → window appears in list_windows even if off-screen."""
+ result = self.client.call_tool(
+ "launch_app", {"bundle_id": CALCULATOR_BUNDLE}
+ )
+ self.assertFalse(
+ result.get("isError"), f"launch_app failed: {result}"
+ )
+ pid = result["structuredContent"]["pid"]
+ self.assertIsInstance(pid, int)
+ self.assertGreater(pid, 0)
+
+ # Give Calculator a moment to create its window.
+ time.sleep(1.0)
+
+ # list_windows (no filter) must include Calculator's window.
+ all_windows = self.client.call_tool("list_windows", {})[
+ "structuredContent"
+ ]["windows"]
+ calc_windows = [w for w in all_windows if w["pid"] == pid]
+ self.assertGreater(
+ len(calc_windows),
+ 0,
+ f"Calculator (pid {pid}) window not found in list_windows",
+ )
+
+ # Every returned record must have the required fields.
+ required = {"window_id", "pid", "app_name", "bounds", "layer", "z_index", "is_on_screen"}
+ for w in calc_windows:
+ missing = required - set(w.keys())
+ self.assertFalse(missing, f"window record missing fields: {missing}")
+
+ def test_screenshot_succeeds_for_hidden_window(self) -> None:
+ """screenshot tool captures the backing store even when is_on_screen=false."""
+ result = self.client.call_tool(
+ "launch_app", {"bundle_id": CALCULATOR_BUNDLE}
+ )
+ self.assertFalse(result.get("isError"), f"launch_app failed: {result}")
+ pid = result["structuredContent"]["pid"]
+ time.sleep(1.0)
+
+ # Grab a window_id for Calculator.
+ windows_result = self.client.call_tool("list_windows", {"pid": pid})[
+ "structuredContent"
+ ]["windows"]
+ self.assertGreater(
+ len(windows_result), 0, "No windows found for Calculator"
+ )
+ window_id = windows_result[0]["window_id"]
+
+ # screenshot must succeed regardless of is_on_screen.
+ shot = self.client.call_tool("screenshot", {"window_id": window_id})
+ self.assertFalse(
+ shot.get("isError"),
+ f"screenshot failed for window_id {window_id}: {shot}",
+ )
+ # Result should contain an image content block.
+ content = shot.get("content", [])
+ has_image = any(c.get("type") == "image" for c in content)
+ has_text = any(c.get("type") == "text" for c in content)
+ self.assertTrue(
+ has_image or has_text,
+ f"screenshot returned no content blocks: {content}",
+ )
+
+ def test_list_windows_pid_filter_returns_warning_on_unknown_pid(self) -> None:
+ """list_windows with a nonexistent pid returns a warning, not an error."""
+ # Use a pid that is virtually guaranteed not to exist.
+ fake_pid = 99999
+ result = self.client.call_tool("list_windows", {"pid": fake_pid})
+ # Must NOT be isError — the tool surfaces a warning in the text body.
+ self.assertFalse(
+ result.get("isError"),
+ "list_windows should not set isError for empty pid filter results",
+ )
+ text_content = " ".join(
+ c.get("text", "") for c in result.get("content", []) if c.get("type") == "text"
+ )
+ self.assertIn(
+ "No windows found",
+ text_content,
+ f"Expected warning text in response, got: {text_content!r}",
+ )
+
+ def test_list_windows_pid_filter_includes_frontmost_hint(self) -> None:
+ """Warning message for empty pid filter includes the frontmost app name."""
+ fake_pid = 99999
+ result = self.client.call_tool("list_windows", {"pid": fake_pid})
+ text_content = " ".join(
+ c.get("text", "") for c in result.get("content", []) if c.get("type") == "text"
+ )
+ # The frontmost-app hint is conditional on having any on-screen window;
+ # on a normal macOS machine this is always true (Finder, at minimum).
+ # We just check the warning structure is present.
+ self.assertIn(str(fake_pid), text_content)
+
+
+if __name__ == "__main__":
+ unittest.main()