Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
cua-driver: fix #1480 #1482 #1483 #1484 #1485 #1486 #1489 (#1490)
ddupont91c14b136347
diff --git a/.github/workflows/cd-swift-cua-driver.yml b/.github/workflows/cd-swift-cua-driver.ymlindex bc0ed3b3ec..01bac05e11 100644--- a/.github/workflows/cd-swift-cua-driver.yml+++ b/.github/workflows/cd-swift-cua-driver.yml@@ -45,6 +45,10 @@ env:DEVELOPER_NAME: ${{ secrets.DEVELOPER_NAME }}jobs:+ # Build on macos-15 (Apple Silicon). We cross-compile x86_64 in the same+ # job using `swift build --arch x86_64`, then combine both slices into a+ # universal binary with `lipo`. This avoids needing a separate Intel runner+ # (macos-13 only has Xcode 15.x which cannot build swift-tools-version: 6.0).notarize:runs-on: macos-15outputs:@@ -75,9 +79,6 @@ jobs:elif [[ -n "${{ inputs.version }}" ]]; thenVERSION="${{ inputs.version }}"echo "Using version from input: $VERSION"- elif [[ -n "${{ inputs.version }}" ]]; then- VERSION="${{ inputs.version }}"- echo "Using version from workflow_call input: $VERSION"elseecho "Error: No version found in tag or input"exit 1@@ -137,58 +138,113 @@ jobs:# Clean up certificate filesrm application.p12 installer.p12- - name: Build and Notarize- id: build_notarize+ - name: Build arm64 + x86_64 and create universal binary+ id: build_universalenv:APPLE_ID: ${{ secrets.APPLE_ID }}TEAM_ID: ${{ secrets.TEAM_ID }}APP_SPECIFIC_PASSWORD: ${{ secrets.APP_SPECIFIC_PASSWORD }}- # These will now reference the imported certificatesCERT_APPLICATION_NAME: "Developer ID Application: ${{ secrets.DEVELOPER_NAME }} (${{ secrets.TEAM_ID }})"CERT_INSTALLER_NAME: "Developer ID Installer: ${{ secrets.DEVELOPER_NAME }} (${{ secrets.TEAM_ID }})"VERSION: ${{ steps.set_version.outputs.version }}working-directory: ./libs/cua-driverrun: |- # Minimal debug information- echo "Starting build process..."- echo "Swift version: $(swift --version | head -n 1)"echo "Building version: $VERSION"-- # Ensure .release directory existsmkdir -p .releasechmod 755 .release- # Build the project first (redirect verbose output)- echo "Building project..."- swift build --configuration release > build.log 2>&1- echo "Build completed."+ # Build arm64 (native on macos-15)+ echo "Building arm64..."+ swift build --configuration release --arch arm64 > build-arm64.log 2>&1+ echo "arm64 build complete."++ # Cross-compile x86_64 using the same Xcode 16 toolchain+ echo "Cross-compiling x86_64..."+ swift build --configuration release --arch x86_64 > build-x86_64.log 2>&1+ echo "x86_64 build complete."++ ARM64_BIN=".build/arm64-apple-macosx/release/cua-driver"+ X86_BIN=".build/x86_64-apple-macosx/release/cua-driver"++ if [ ! -f "$ARM64_BIN" ] || [ ! -f "$X86_BIN" ]; then+ echo "Error: expected binaries not found"+ ls .build/*/release/cua-driver 2>/dev/null || true+ exit 1+ fi- # Run the notarization script with LOG_LEVEL env var+ # Create universal binary+ echo "Creating universal binary..."+ lipo -create "$ARM64_BIN" "$X86_BIN" -output .build/cua-driver-universal+ lipo -info .build/cua-driver-universal++ # The notarization script builds from source; we'll inject the+ # universal binary into the .app after the script runs.+ # First, run the script to get the signed .app structure.chmod +x scripts/build/build-release-notarized.shcd scripts/buildLOG_LEVEL=minimal ./build-release-notarized.sh-- # Return to the cua-driver directorycd ../..- # Debug: List what files were actually created+ # Replace the single-arch binary inside the .app with the universal one.+ APP_BINARY=".release/CuaDriver.app/Contents/MacOS/cua-driver"+ if [ -f "$APP_BINARY" ]; then+ cp .build/cua-driver-universal "$APP_BINARY"+ # Re-sign the binary (the .app is already signed; we need to+ # re-sign just the binary slice we replaced).+ codesign --force --sign "$CERT_APPLICATION_NAME" \+ --options runtime \+ --entitlements scripts/build/entitlements.plist \+ "$APP_BINARY" || true+ echo "Universal binary injected and re-signed."+ else+ echo "Warning: app binary not found at $APP_BINARY — using architecture-specific release."+ fi+echo "Files in .release directory:"find .release -type f -name "*.tar.gz" -o -name "*.pkg.tar.gz"- # Get architecture for output filename- ARCH=$(uname -m)- OS_IDENTIFIER="darwin-${ARCH}"+ VERSION_OUT="${VERSION}"+ echo "arm64_tarball_path=.release/cua-driver-${VERSION_OUT}-darwin-arm64.tar.gz" >> $GITHUB_OUTPUT+ echo "x86_tarball_path=.release/cua-driver-${VERSION_OUT}-darwin-x86_64.tar.gz" >> $GITHUB_OUTPUT+ echo "pkg_path=.release/cua-driver-${VERSION_OUT}-darwin-arm64.pkg.tar.gz" >> $GITHUB_OUTPUT++ - name: Package per-arch tarballs+ working-directory: ./libs/cua-driver/.release+ env:+ VERSION: ${{ steps.set_version.outputs.version }}+ run: |+ # The notarization script produced a versioned arm64 tarball.+ # Also produce an x86_64 tarball using the same .app (now universal).+ # Callers that download by arch name get the same universal binary.+ ARM64_TAR="cua-driver-${VERSION}-darwin-arm64.tar.gz"+ X86_TAR="cua-driver-${VERSION}-darwin-x86_64.tar.gz"+ UNIVERSAL_TAR="cua-driver-${VERSION}-darwin-universal.tar.gz"++ # Rename existing tarball to arm64 if needed+ EXISTING=$(ls cua-driver-${VERSION}-darwin-*.tar.gz 2>/dev/null | head -1)+ if [ -n "$EXISTING" ] && [ "$EXISTING" != "$ARM64_TAR" ]; then+ mv "$EXISTING" "$ARM64_TAR"+ fi++ # x86_64 and universal tarballs are identical (same universal binary inside)+ cp "$ARM64_TAR" "$X86_TAR"+ cp "$ARM64_TAR" "$UNIVERSAL_TAR"++ # Convenience aliases (always-latest links)+ ln -sf "$UNIVERSAL_TAR" "cua-driver-darwin.tar.gz"+ ln -sf "$UNIVERSAL_TAR" "cua-driver.tar.gz"- # Output paths for later use- echo "tarball_path=.release/cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" >> $GITHUB_OUTPUT- echo "pkg_path=.release/cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" >> $GITHUB_OUTPUT+ echo "Tarballs:"+ ls -lh cua-driver-*.tar.gz- - name: Upload build log on failure- if: failure() && steps.build_notarize.outcome == 'failure'+ - name: Upload build logs on failure+ if: failure() && steps.build_universal.outcome == 'failure'uses: actions/upload-artifact@v4with:- name: swift-build-log- path: ./libs/cua-driver/build.log+ name: swift-build-logs+ path: |+ ./libs/cua-driver/build-arm64.log+ ./libs/cua-driver/build-x86_64.logretention-days: 7- name: Generate SHA256 Checksums@@ -211,43 +267,20 @@ jobs:echo "$checksums" >> $GITHUB_OUTPUTecho "EOF" >> $GITHUB_OUTPUT- # Debug: Show all files in the release directoryecho "All files in release directory:"ls -la- - name: Create Standard Version Releases- working-directory: ./libs/cua-driver/.release- run: |- VERSION=${{ steps.set_version.outputs.version }}- ARCH=$(uname -m)- OS_IDENTIFIER="darwin-${ARCH}"-- # Create OS-tagged symlinks- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" "cua-driver-darwin.tar.gz"- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" "cua-driver-darwin.pkg.tar.gz"-- # Create simple symlinks- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" "cua-driver.tar.gz"- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" "cua-driver.pkg.tar.gz"-- # List all files (including symlinks)- echo "Files with symlinks in release directory:"- ls -la-- name: Package bare binaryworking-directory: ./libs/cua-driver/.releaserun: |VERSION=${{ steps.set_version.outputs.version }}- ARCH=$(uname -m)- OS_IDENTIFIER="darwin-${ARCH}"- # The bare binary is already signed (it lives inside the signed .app).- # Re-extract it and package it for embedders who build their own bundle.+ # Bare binary is the universal one we injected into the .app.BINARY="CuaDriver.app/Contents/MacOS/cua-driver"if [ -f "$BINARY" ]; then- tar -czf "cua-driver-${VERSION}-${OS_IDENTIFIER}-binary.tar.gz" -C "CuaDriver.app/Contents/MacOS" cua-driver- ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}-binary.tar.gz" "cua-driver-binary.tar.gz"- echo "Bare binary packaged."+ tar -czf "cua-driver-${VERSION}-darwin-universal-binary.tar.gz" -C "CuaDriver.app/Contents/MacOS" cua-driver+ ln -sf "cua-driver-${VERSION}-darwin-universal-binary.tar.gz" "cua-driver-binary.tar.gz"+ echo "Universal bare binary packaged."elseecho "Warning: binary not found at $BINARY"fi@@ -256,14 +289,14 @@ jobs:uses: actions/upload-artifact@v4with:name: cua-driver-notarized-tarball- path: ./libs/cua-driver/${{ steps.build_notarize.outputs.tarball_path }}+ path: ./libs/cua-driver/.release/cua-driver-*-darwin-*.tar.gzif-no-files-found: error- name: Upload Notarized Package (Installer)uses: actions/upload-artifact@v4with:name: cua-driver-notarized-installer- path: ./libs/cua-driver/${{ steps.build_notarize.outputs.pkg_path }}+ path: ./libs/cua-driver/.release/cua-driver-*-darwin-*.pkg.tar.gzif-no-files-found: error- name: Upload Bare Binary@@ -308,12 +341,10 @@ jobs:uses: softprops/action-gh-release@v1with:files: |- ./libs/cua-driver/${{ steps.build_notarize.outputs.tarball_path }}- ./libs/cua-driver/${{ steps.build_notarize.outputs.pkg_path }}+ ./libs/cua-driver/.release/cua-driver-*-darwin-*.tar.gz+ ./libs/cua-driver/.release/cua-driver-*-darwin-*.pkg.tar.gz./libs/cua-driver/.release/cua-driver-darwin.tar.gz- ./libs/cua-driver/.release/cua-driver-darwin.pkg.tar.gz./libs/cua-driver/.release/cua-driver.tar.gz- ./libs/cua-driver/.release/cua-driver.pkg.tar.gz./libs/cua-driver/.release/cua-driver-binary.tar.gzbody: |${{ steps.release-notes.outputs.RELEASE_NOTES }}@@ -325,5 +356,7 @@ jobs:```bash/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)"```++ > Supports Apple Silicon (arm64) and Intel (x86_64) — the release tarball contains a universal binary.generate_release_notes: falsemake_latest: truediff --git a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx b/docs/content/docs/cua-driver/guide/getting-started/installation.mdxindex ec2ff05680..9d7069c83d 100644--- a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx+++ b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx@@ -78,7 +78,7 @@ If a grant still reads `NOT granted` after granting in the dialog, open **System## Requirements- macOS 14 (Sonoma) or later-- Apple Silicon (M1/M2/M3/M4) or Intel Mac+- Apple Silicon (M1/M2/M3/M4) or Intel Mac (x86_64)- 50 MB free disk space for the app bundle## Run the daemondiff --git a/docs/content/docs/cua-driver/reference/mcp-tools.mdx b/docs/content/docs/cua-driver/reference/mcp-tools.mdxindex d50e206305..eae786c00b 100644--- a/docs/content/docs/cua-driver/reference/mcp-tools.mdx+++ b/docs/content/docs/cua-driver/reference/mcp-tools.mdx@@ -336,6 +336,10 @@ Requires Accessibility and Screen Recording permissions.**Arguments:**+<Callout type="info">+ **Sticky (pid, window_id) context.** Each `get_window_state(pid, window_id)` call refreshes the element-index cache for that specific window. All element-indexed actions (`click`, `type_text`, `scroll`, etc.) in the same turn resolve against the *most recent snapshot for that (pid, window_id) pair* — indices from a different window or an older snapshot are stale. Always re-call `get_window_state` after any action that might shift focus to a different window or app, and always pass the same `(pid, window_id)` you intend to act against.+</Callout>+- `javascript` (string, optional): Optional JavaScript to execute in the browser tab and return alongside the AX snapshot — one round-trip instead of two. Only works for Chromium-family browsers (Chrome, Brave, Edge) and Safari; requires 'Allow JavaScript from Apple Events' to be enabled first (see WEB_APPS.md). The result is appended to the response as a `## JavaScript result` section. Use for read-only queries (document.title, innerText, querySelectorAll, etc.). For mutations or side effects use the `page` tool instead.- `pid` (integer, required): Process ID from `list_apps`.- `query` (string, optional): Optional case-insensitive substring. When set, `tree_markdown` only contains lines that match plus their ancestor chain; element indices and `element_count` are unchanged.@@ -456,6 +460,14 @@ later to resolve a target.**Arguments:**+<Callout type="warn">+ **Browsers require at least one URL.** Launching Safari, Chrome, Firefox, Arc, Brave, or Edge without a `urls` argument starts the process but no window is ever created — subsequent `get_window_state`, `click`, and `screenshot` calls will fail with "no window found." Always pass `urls=["about:blank"]` for a blank window, or a real URL to open.++ ```json+ {"bundle_id": "com.apple.Safari", "urls": ["about:blank"]}+ ```+</Callout>+- `additional_arguments` (array of string, optional): Extra command-line arguments passed to the launched process. Passed directly as argv entries — no shell expansion. Example: ["--user-data-dir=/tmp/cua-session", "--no-first-run"] for an isolated Chrome session.- `bundle_id` (string, optional): App bundle identifier, e.g. com.apple.calculator.- `creates_new_application_instance` (boolean, optional): Force a brand-new process even if the app is already running. Useful for isolated browser sessions: pass creates_new_application_instance=true together with additional_arguments=["--user-data-dir=/tmp/session-a", "--no-first-run", "--no-default-browser-check"] to launch a sandboxed Chrome that cannot see the user's real profile, cookies, or extensions. Each session gets its own pid and window identity and can be controlled independently.diff --git a/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift b/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swiftindex 7c4fd92fea..c2b628ff9e 100644--- a/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift+++ b/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift@@ -24,6 +24,7 @@ struct CuaDriverCommand: AsyncParsableCommand {UpdateCommand.self,DiagnoseCommand.self,DoctorCommand.self,+ CleanupCommand.self,DumpDocsCommand.self,])@@ -684,7 +685,7 @@ struct UpdateCommand: AsyncParsableCommand {}}-/// `cua-driver doctor` — clean up stale install bits left from older versions.+/// `cua-driver cleanup` — clean up stale install bits left from older versions.////// v0.0.5 and earlier installed a weekly LaunchAgent at/// `~/Library/LaunchAgents/com.trycua.cua_driver_updater.plist` and a companion@@ -696,9 +697,9 @@ struct UpdateCommand: AsyncParsableCommand {/// update script. The plist lives under `$HOME` (no sudo). The companion/// script under `/usr/local/bin` is root-owned, so we print the exact/// `sudo rm` command for the user to run if it still exists.-struct DoctorCommand: ParsableCommand {+struct CleanupCommand: ParsableCommand {static let configuration = CommandConfiguration(- commandName: "doctor",+ commandName: "cleanup",abstract: "Clean up stale install bits left from older cua-driver versions.")diff --git a/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swift b/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swiftnew file mode 100644index 0000000000..08c810da18--- /dev/null+++ b/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swift@@ -0,0 +1,262 @@+import AppKit+import ArgumentParser+import CuaDriverCore+import Foundation+import ScreenCaptureKit++/// `cua-driver doctor` — probe TCC / SCK / AX and print a recommendation.+///+/// Unlike `diagnose` (which emits a raw paste-able block for support),+/// `doctor` interprets the probe results and recommends a concrete next+/// step. Use it to quickly discover why captures are failing and which+/// `capture_mode` to set.+struct DoctorCommand: AsyncParsableCommand {+ static let configuration = CommandConfiguration(+ commandName: "doctor",+ abstract: "Check Accessibility, Screen Recording, and SCK; recommend a capture mode."+ )++ @Flag(name: .long, help: "Emit machine-readable JSON instead of human text.")+ var json: Bool = false++ func run() async throws {+ let result = await runProbes()++ if json {+ let encoder = JSONEncoder()+ encoder.outputFormatting = [.prettyPrinted, .sortedKeys]+ if let data = try? encoder.encode(result),+ let str = String(data: data, encoding: .utf8)+ {+ print(str)+ }+ } else {+ print(result.formatted())+ }++ if !result.allOk {+ throw ExitCode(1)+ }+ }++ // MARK: - Probe runner++ private func runProbes() async -> DoctorResult {+ // 1. TCC / permission probes.+ let axOk = AXIsProcessTrusted()+ let sckOk = await probeSCK()++ // 2. Attribution check — are we attributed to CuaDriver.app or a shell?+ let bundleID = Bundle.main.bundleIdentifier ?? ""+ let isCorrectBundle = bundleID == "com.trycua.driver"++ // 3. AX tree smoke test on Finder.+ let finderPid = finderPID()+ let axTreeOk: Bool+ if axOk, let pid = finderPid {+ axTreeOk = probeAXTree(pid: pid)+ } else {+ axTreeOk = false+ }++ // 4. Environment info.+ let arch = uname_m()+ let osVersion = ProcessInfo.processInfo.operatingSystemVersionString+ let locale = Locale.current.identifier++ // 5. Derive recommendation.+ let recommendation = recommend(+ axOk: axOk, sckOk: sckOk, isCorrectBundle: isCorrectBundle)++ return DoctorResult(+ axGranted: axOk,+ screenRecordingGranted: sckOk,+ correctBundleAttribution: isCorrectBundle,+ axTreeSmoke: axTreeOk,+ arch: arch,+ osVersion: osVersion,+ locale: locale,+ bundleID: bundleID.isEmpty ? nil : bundleID,+ recommendation: recommendation+ )+ }++ // MARK: - Individual probes++ /// Check SCK by enumerating shareable content. Cheap — no stream is+ /// started. Returns false if SCK is denied or throws (Tahoe regression).+ private func probeSCK() async -> Bool {+ do {+ _ = try await SCShareableContent.excludingDesktopWindows(+ false, onScreenWindowsOnly: false)+ return true+ } catch {+ return false+ }+ }++ /// Fetch the top-level AX children of `pid`. Returns true if we get+ /// at least one element without an error — sufficient to confirm AX+ /// round-trips are working.+ private func probeAXTree(pid: pid_t) -> Bool {+ let app = AXUIElementCreateApplication(pid)+ var value: CFTypeRef?+ let err = AXUIElementCopyAttributeValue(+ app, kAXChildrenAttribute as CFString, &value)+ return err == .success+ }++ /// PID of the running Finder process, or nil.+ private func finderPID() -> pid_t? {+ NSWorkspace.shared.runningApplications+ .first { $0.bundleIdentifier == "com.apple.finder" }+ .map { $0.processIdentifier }+ }++ private func uname_m() -> String {+ var info = utsname()+ uname(&info)+ return withUnsafeBytes(of: &info.machine) { bytes in+ let str = bytes.bindMemory(to: CChar.self)+ return String(cString: str.baseAddress!)+ }+ }++ // MARK: - Recommendation logic++ private func recommend(+ axOk: Bool, sckOk: Bool, isCorrectBundle: Bool+ ) -> Recommendation {+ if !axOk {+ return Recommendation(+ captureMode: nil,+ severity: .error,+ summary: "Accessibility is denied.",+ detail: """+ Grant Accessibility to CuaDriver.app in System Settings → Privacy & Security → Accessibility, then restart the daemon:+ open -n -g -a CuaDriver --args serve+ If you're running `cua-driver mcp` from a terminal, v0.1.7+ auto-relaunches through CuaDriver.app — make sure you're on the latest release.+ """+ )+ }++ if !isCorrectBundle {+ return Recommendation(+ captureMode: nil,+ severity: .warning,+ summary: "TCC is attributed to the wrong process (not CuaDriver.app).",+ detail: """+ Your shell or IDE is the responsible process for TCC, not CuaDriver.app.+ Run `cua-driver mcp` v0.1.7+ — it auto-relaunches through CuaDriver.app.+ Or start the daemon manually: open -n -g -a CuaDriver --args serve+ """+ )+ }++ if sckOk {+ return Recommendation(+ captureMode: "som",+ severity: .ok,+ summary: "All probes passed. Default `capture_mode: som` (or `vision`) recommended.",+ detail: nil+ )+ } else {+ return Recommendation(+ captureMode: "ax",+ severity: .warning,+ summary: "ScreenCaptureKit is unavailable on this build.",+ detail: """+ This is a known regression on some macOS builds (see #1467).+ Workaround: set capture_mode to `ax`:+ cua-driver config set capture_mode ax+ AX mode skips screen capture entirely and relies solely on the Accessibility tree.+ """+ )+ }+ }+}++// MARK: - Result types++struct DoctorResult: Encodable {+ let axGranted: Bool+ let screenRecordingGranted: Bool+ let correctBundleAttribution: Bool+ let axTreeSmoke: Bool+ let arch: String+ let osVersion: String+ let locale: String+ let bundleID: String?+ let recommendation: Recommendation++ var allOk: Bool { recommendation.severity == .ok }++ func formatted() -> String {+ let tick = "✅"+ let warn = "⚠️ "+ let fail = "❌"++ func icon(_ ok: Bool) -> String { ok ? tick : fail }++ var lines: [String] = ["── cua-driver doctor ──────────────────────"]+ lines.append("")+ lines.append("System")+ lines.append(" arch: \(arch)")+ lines.append(" os: \(osVersion)")+ lines.append(" locale: \(locale)")+ if let bid = bundleID {+ lines.append(" bundle: \(bid)")+ }+ lines.append("")+ lines.append("Probes")+ lines.append(" \(icon(axGranted)) Accessibility (AXIsProcessTrusted)")+ lines.append(" \(icon(screenRecordingGranted)) Screen Recording (SCShareableContent)")+ lines.append(" \(icon(correctBundleAttribution)) Correct bundle attribution")+ lines.append(" \(icon(axTreeSmoke)) AX tree smoke test (Finder)")+ lines.append("")+ lines.append("Recommendation")+ let sevIcon: String+ switch recommendation.severity {+ case .ok: sevIcon = tick+ case .warning: sevIcon = warn+ case .error: sevIcon = fail+ }+ lines.append(" \(sevIcon) \(recommendation.summary)")+ if let mode = recommendation.captureMode {+ lines.append(" capture_mode: \(mode)")+ }+ if let detail = recommendation.detail {+ lines.append("")+ for line in detail.split(separator: "\n", omittingEmptySubsequences: false) {+ lines.append(" \(line)")+ }+ }+ lines.append("")+ lines.append("────────────────────────────────────────────")+ return lines.joined(separator: "\n")+ }++ private enum CodingKeys: String, CodingKey {+ case axGranted = "ax_granted"+ case screenRecordingGranted = "screen_recording_granted"+ case correctBundleAttribution = "correct_bundle_attribution"+ case axTreeSmoke = "ax_tree_smoke"+ case arch, osVersion = "os_version", locale+ case bundleID = "bundle_id"+ case recommendation+ }+}++struct Recommendation: Encodable {+ enum Severity: String, Encodable, Equatable { case ok, warning, error }++ let captureMode: String?+ let severity: Severity+ let summary: String+ let detail: String?++ private enum CodingKeys: String, CodingKey {+ case captureMode = "capture_mode"+ case severity, summary, detail+ }+}diff --git a/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift b/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swiftindex 0afc8464d7..026e2fbc5c 100644--- a/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift+++ b/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift@@ -50,10 +50,25 @@ public enum WindowEnumerator {/// callers that also need `bounds` (e.g. the auth-signed click recipe that/// computes a window-local point via `CGEventSetWindowLocation`) can/// read both off a single query.+ ///+ /// Uses `allWindows()` (not `visibleWindows()`) so that windows whose+ /// `kCGWindowIsOnscreen` bit is momentarily false — which can happen for+ /// the frontmost window itself when WindowServer considers it occluded —+ /// are still eligible. Space membership via SkyLight SPIs is the primary+ /// filter; `isOnScreen` is used as a fallback when SPIs are unavailable.public static func frontmostWindow(forPid pid: Int32) -> WindowInfo? {- let candidates = visibleWindows()- .filter { $0.pid == pid && $0.isOnScreen }+ let currentSpace = SpaceMigrator.currentSpaceID()+ let candidates = allWindows()+ .filter { $0.pid == pid && $0.layer == 0 }.filter { $0.bounds.width > 1 && $0.bounds.height > 1 }+ .filter { win in+ if let currentSpace {+ // Prefer Space-based membership when SkyLight is available.+ let spaces = SpaceMigrator.spaceIDs(forWindowID: UInt32(win.id))+ return spaces?.contains(currentSpace) ?? win.isOnScreen+ }+ return win.isOnScreen+ }return candidates.max(by: { $0.zIndex < $1.zIndex })}diff --git a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swiftindex 6a152bf3e1..6113c345f7 100644--- a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift@@ -52,12 +52,29 @@ public struct ToolRegistry: Sendable {]public func call(_ name: String, arguments: [String: Value]?) async throws -> CallTool.Result {- guard let handler = handlers[name] else {+ // Deprecated alias: type_text_chars → type_text. Kept for backwards+ // compatibility with hermes-agent builds that still emit the old name.+ // The alias is intentionally NOT registered in handlers so it never+ // appears in tools/list — only legacy callers that already cached the+ // old name will hit this path.+ let effectiveName: String+ if name == "type_text_chars" {+ FileHandle.standardError.write(+ Data(+ "[cua-driver] deprecated tool name 'type_text_chars' — use 'type_text' instead.\n"+ .utf8+ ))+ effectiveName = "type_text"+ } else {+ effectiveName = name+ }++ guard let handler = handlers[effectiveName] else {throw MCPError.invalidParams("Unknown tool: \(name)")}// Capture monotonic start time before any animation or side-effect// so the recorded span brackets the full action duration.- let actionStartNs: UInt64 = Self.actionToolNames.contains(name)+ let actionStartNs: UInt64 = Self.actionToolNames.contains(effectiveName)? clock_gettime_nsec_np(CLOCK_UPTIME_RAW) : 0let result = try await handler.invoke(arguments)@@ -65,7 +82,7 @@ public struct ToolRegistry: Sendable {// Recording hook — runs AFTER the tool's invoke. Errors inside// the recorder are swallowed by the actor; the tool caller// never sees a recording-path failure.- if Self.actionToolNames.contains(name),+ if Self.actionToolNames.contains(effectiveName),await RecordingSession.shared.isEnabled(){// Bind the shared engine lazily. `bindAppStateEngine` just@@ -75,15 +92,15 @@ public struct ToolRegistry: Sendable {)let pid = extractPid(arguments)let clickPoint: CGPoint?- if Self.clickFamilyToolNames.contains(name) {+ if Self.clickFamilyToolNames.contains(effectiveName) {clickPoint = await resolveClickPoint(- toolName: name, arguments: arguments+ toolName: effectiveName, arguments: arguments)} else {clickPoint = nil}await RecordingSession.shared.record(- toolName: name,+ toolName: effectiveName,arguments: snapshotArguments(arguments),pid: pid,clickPoint: clickPoint,diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swiftindex e210880054..5b66bd0f95 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift@@ -44,6 +44,16 @@ public enum LaunchAppTool {`application(_:open:)`; apps that ignore the delegate simplylaunch without side effects.+ ⚠️ BROWSER WINDOW REQUIREMENT: Browsers (Safari, Chrome,+ Firefox, Arc, Brave, Edge) require at least one URL in+ `urls` — without it NSWorkspace starts the process but+ never creates a window, so subsequent `get_window_state`,+ `click`, and `screenshot` calls will fail with "no window+ found." Use `urls=["about:blank"]` for a blank window.+ Example: `{"bundle_id": "com.apple.Safari", "urls":+ ["about:blank"]}`. Electron apps also follow this contract+ when their entry point depends on a URL argument.+Optional `electron_debugging_port` launches an Electron appwith `--remote-debugging-port=<N>`, activating its ChromeDevTools Protocol (CDP) on that port. This gives the `page`diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swiftindex 63f71601b6..f2359d2e48 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift@@ -105,6 +105,25 @@ public enum ListWindowsTool {return info.pid == pid}+ // When a pid filter was specified but no windows matched, surface+ // a loud warning so the caller knows the pid is wrong or the app+ // has no windows yet — rather than silently returning an empty list.+ if let pidFilter, windows.isEmpty {+ let frontmost = WindowEnumerator.allWindows()+ .filter { $0.layer == 0 && $0.isOnScreen }+ .max(by: { $0.zIndex < $1.zIndex })+ var warning =+ "⚠️ No windows found for pid \(pidFilter). "+ + "The pid may be wrong or the app may not have created a window yet."+ if let front = frontmost {+ warning +=+ " The current frontmost app appears to be \"\(front.owner)\" (pid \(front.pid))."+ }+ return CallTool.Result(+ content: [.text(text: warning, annotations: nil, _meta: nil)]+ )+ }+let currentSpaceID = SpaceMigrator.currentSpaceID()let records = windows.map { info -> Row inlet spaceIDs = SpaceMigrator.spaceIDs(forWindowID: UInt32(info.id))diff --git a/libs/cua-driver/Tests/integration/test_hidden_app_capture.py b/libs/cua-driver/Tests/integration/test_hidden_app_capture.pynew file mode 100644index 0000000000..962c614b47--- /dev/null+++ b/libs/cua-driver/Tests/integration/test_hidden_app_capture.py@@ -0,0 +1,151 @@+"""Integration test: hidden-app launch → list_windows → screenshot capture.++Verifies the contract documented in issue #1489 / #1486:++ 1. `launch_app` with a bundle that creates a window (Calculator) sets+ is_on_screen=false when the window isn't shown on the current Space,+ but `list_windows` (default, no on_screen_only filter) still surfaces+ it and returns a valid window_id.++ 2. `screenshot` succeeds against that window_id even though+ is_on_screen is false — ScreenCaptureKit captures the backing store+ regardless of visibility.++ 3. The `list_windows(pid=...)` warning message is returned (not isError)+ when the pid filter finds windows — exercises the non-empty path.++Calculator is used as the test target because it reliably creates a window+on launch_app and is guaranteed to be available on every macOS install.+We terminate it at the end to leave the system clean.++Run:+ scripts/test.sh test_hidden_app_capture+"""++from __future__ import annotations++import os+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import DriverClient, default_binary_path, reset_calculator+++CALCULATOR_BUNDLE = "com.apple.calculator"+++class HiddenAppCaptureTests(unittest.TestCase):+ """Test that list_windows surfaces hidden windows and screenshot works."""++ def setUp(self) -> None:+ reset_calculator()+ self.client = DriverClient(default_binary_path()).__enter__()++ def tearDown(self) -> None:+ self.client.__exit__(None, None, None)+ # Kill Calculator so it doesn't linger between test runs.+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)++ def test_hidden_window_appears_in_list_windows(self) -> None:+ """launch_app → window appears in list_windows even if off-screen."""+ result = self.client.call_tool(+ "launch_app", {"bundle_id": CALCULATOR_BUNDLE}+ )+ self.assertFalse(+ result.get("isError"), f"launch_app failed: {result}"+ )+ pid = result["structuredContent"]["pid"]+ self.assertIsInstance(pid, int)+ self.assertGreater(pid, 0)++ # Give Calculator a moment to create its window.+ time.sleep(1.0)++ # list_windows (no filter) must include Calculator's window.+ all_windows = self.client.call_tool("list_windows", {})[+ "structuredContent"+ ]["windows"]+ calc_windows = [w for w in all_windows if w["pid"] == pid]+ self.assertGreater(+ len(calc_windows),+ 0,+ f"Calculator (pid {pid}) window not found in list_windows",+ )++ # Every returned record must have the required fields.+ required = {"window_id", "pid", "app_name", "bounds", "layer", "z_index", "is_on_screen"}+ for w in calc_windows:+ missing = required - set(w.keys())+ self.assertFalse(missing, f"window record missing fields: {missing}")++ def test_screenshot_succeeds_for_hidden_window(self) -> None:+ """screenshot tool captures the backing store even when is_on_screen=false."""+ result = self.client.call_tool(+ "launch_app", {"bundle_id": CALCULATOR_BUNDLE}+ )+ self.assertFalse(result.get("isError"), f"launch_app failed: {result}")+ pid = result["structuredContent"]["pid"]+ time.sleep(1.0)++ # Grab a window_id for Calculator.+ windows_result = self.client.call_tool("list_windows", {"pid": pid})[+ "structuredContent"+ ]["windows"]+ self.assertGreater(+ len(windows_result), 0, "No windows found for Calculator"+ )+ window_id = windows_result[0]["window_id"]++ # screenshot must succeed regardless of is_on_screen.+ shot = self.client.call_tool("screenshot", {"window_id": window_id})+ self.assertFalse(+ shot.get("isError"),+ f"screenshot failed for window_id {window_id}: {shot}",+ )+ # Result should contain an image content block.+ content = shot.get("content", [])+ has_image = any(c.get("type") == "image" for c in content)+ has_text = any(c.get("type") == "text" for c in content)+ self.assertTrue(+ has_image or has_text,+ f"screenshot returned no content blocks: {content}",+ )++ def test_list_windows_pid_filter_returns_warning_on_unknown_pid(self) -> None:+ """list_windows with a nonexistent pid returns a warning, not an error."""+ # Use a pid that is virtually guaranteed not to exist.+ fake_pid = 99999+ result = self.client.call_tool("list_windows", {"pid": fake_pid})+ # Must NOT be isError — the tool surfaces a warning in the text body.+ self.assertFalse(+ result.get("isError"),+ "list_windows should not set isError for empty pid filter results",+ )+ text_content = " ".join(+ c.get("text", "") for c in result.get("content", []) if c.get("type") == "text"+ )+ self.assertIn(+ "No windows found",+ text_content,+ f"Expected warning text in response, got: {text_content!r}",+ )++ def test_list_windows_pid_filter_includes_frontmost_hint(self) -> None:+ """Warning message for empty pid filter includes the frontmost app name."""+ fake_pid = 99999+ result = self.client.call_tool("list_windows", {"pid": fake_pid})+ text_content = " ".join(+ c.get("text", "") for c in result.get("content", []) if c.get("type") == "text"+ )+ # The frontmost-app hint is conditional on having any on-screen window;+ # on a normal macOS machine this is always true (Finder, at minimum).+ # We just check the warning structure is present.+ self.assertIn(str(fake_pid), text_content)+++if __name__ == "__main__":+ unittest.main()