Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
20.2 KB
Raw Blame
import AppKit
import ApplicationServices
import ScreenCaptureKit
import CryptoKit

struct AppIdentity {
    let app: NSRunningApplication
    let bundleID: String
    let executable: URL
    let launched: ProcessStart
    // Launch Services may omit launchDate for directly launched apps. The
    // kernel start time still prevents a recycled PID from inheriting a grant.
    struct ProcessStart: Equatable {
        let seconds: UInt64
        let microseconds: UInt64
        init?(pid: pid_t) {
            var info = proc_bsdinfo()
            let size = Int32(MemoryLayout<proc_bsdinfo>.size)
            guard proc_pidinfo(pid, PROC_PIDTBSDINFO, 0, &info, size) == size else { return nil }
            seconds = info.pbi_start_tvsec; microseconds = info.pbi_start_tvusec
        }
    }
    var pid: pid_t { app.processIdentifier }
    var name: String { app.localizedName ?? bundleID }

    static func resolve(_ bundleID: String, pid: pid_t?) throws -> AppIdentity {
        let matches = NSWorkspace.shared.runningApplications.filter {
            $0.bundleIdentifier == bundleID && $0.activationPolicy == .regular && (pid == nil || $0.processIdentifier == pid)
        }
        guard matches.count == 1, let app = matches.first, let executable = app.executableURL, let launched = ProcessStart(pid: app.processIdentifier) else {
            throw UseError("app_unavailable", "Choose an exact running app from computer_discover; supply its pid if several copies are open.", next: "discover")
        }
        guard isAllowed(app) else { throw UseError("protected_app", "This app cannot be operated through Computer Use.", next: "use_structured_tool") }
        return AppIdentity(app: app, bundleID: bundleID, executable: executable, launched: launched)
    }
    @MainActor static func open(_ bundleID: String, pid: pid_t?) async throws -> AppIdentity {
        guard isAllowed(bundleID: bundleID) else { throw UseError("protected_app", "This app cannot be operated through Computer Use.", next: "use_structured_tool") }
        // A PID pins a running instance. Never replace it with a newly launched app.
        if pid != nil || NSRunningApplication.runningApplications(withBundleIdentifier: bundleID).count > 0 {
            let identity = try resolve(bundleID, pid: pid)
            identity.app.unhide()
            return identity
        }
        guard let url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: bundleID),
              Bundle(url: url)?.bundleIdentifier == bundleID else {
            throw UseError("app_unavailable", "This app is not installed. Choose an app from computer_discover.", next: "discover")
        }
        let configuration = NSWorkspace.OpenConfiguration()
        configuration.activates = true
        let launched = try await NSWorkspace.shared.openApplication(at: url, configuration: configuration)
        // Launch Services can complete before the app enters its UI run loop.
        for _ in 0..<30 {
            try Task.checkCancellation()
            if let identity = try? resolve(bundleID, pid: launched.processIdentifier) { return identity }
            if launched.isTerminated { break }
            try await Task.sleep(nanoseconds: 100_000_000)
        }
        return try resolve(bundleID, pid: launched.processIdentifier)
    }
    static func installedApps() -> [[String: Any]] {
        // Enumerate app bundles only, never documents or window content.
        let roots = ["/Applications", "/Applications/Utilities", "/System/Applications", "/System/Applications/Utilities",
                     NSHomeDirectory() + "/Applications"]
        var apps: [String: String] = [:]
        for root in roots {
            for url in (try? FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: root), includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] {
                guard url.pathExtension == "app", let bundle = Bundle(url: url), let id = bundle.bundleIdentifier,
                      isAllowed(bundleID: id) else { continue }
                apps[id] = bundle.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String
                    ?? bundle.object(forInfoDictionaryKey: "CFBundleName") as? String ?? url.deletingPathExtension().lastPathComponent
            }
        }
        return apps.map { ["app_id": $0.key, "name": $0.value] }
    }
    func validate() throws {
        guard !app.isTerminated, let current = NSRunningApplication(processIdentifier: pid),
              current.bundleIdentifier == bundleID, current.executableURL == executable, ProcessStart(pid: pid) == launched,
              Self.isAllowed(current) else {
            throw UseError("app_changed", "The approved app exited or changed. Open a new session.", next: "open_session")
        }
    }
    static func isAllowed(_ app: NSRunningApplication) -> Bool {
        guard let id = app.bundleIdentifier, app.processIdentifier != ProcessInfo.processInfo.processIdentifier else { return false }
        return isAllowed(bundleID: id)
    }
    static func isAllowed(bundleID id: String) -> Bool {
        // These surfaces can change the permission boundary or execute arbitrary commands.
        let protected: Set<String> = ["com.apple.Terminal", "com.googlecode.iterm2", "dev.warp.Warp-Stable",
            "com.mitchellh.ghostty", "com.apple.ScriptEditor2", "com.apple.systempreferences",
            "com.apple.SecurityAgent", "com.apple.loginwindow", "com.apple.keychainaccess",
            "com.apple.Passwords", "com.1password.1password", "com.agilebits.onepassword7"]
        return !protected.contains(id) && !id.hasPrefix("ai.omnirush.desktop")
            && !id.hasPrefix("com.omnirush") && !id.hasPrefix("com.openai")
    }
}

struct WindowTarget {
    let id: CGWindowID
    let title: String
    let element: AXUIElement
    let frame: CGRect
}

struct ElementRecord {
    let ref: String
    let element: AXUIElement
    let role: String
    let label: String
    let value: String?
    let frame: CGRect
    let actions: [String]
    let settable: Bool
    let enabled: Bool
    let protected: Bool
}

struct WindowState {
    let records: [ElementRecord]
    let visited: Int
    let truncated: Bool
    let protectedFrames: [CGRect]
}

@MainActor
final class MacAccessibility {
    func windows(_ app: AppIdentity) async throws -> [WindowTarget] {
        try requirePermissions()
        try app.validate()
        let root = AXUIElementCreateApplication(app.pid)
        AXUIElementSetMessagingTimeout(root, 0.2)
        // Electron exposes its accessibility tree on demand. This is its
        // documented assistive-technology handshake, not a macOS permission grant.
        // Native apps can reject the optional attribute and still expose windows.
        if (attribute(root, "AXManualAccessibility") as? Bool) == false,
           AXUIElementSetAttributeValue(root, "AXManualAccessibility" as CFString, kCFBooleanTrue) == .success {
            // Electron debounces activation for two seconds. Do not keep setting
            // the attribute during retries: that restarts its countdown.
            let deadline = ProcessInfo.processInfo.systemUptime + 3
            while ProcessInfo.processInfo.systemUptime < deadline {
                try app.validate()
                if (attribute(root, "AXManualAccessibility") as? Bool) == true { break }
                try await Task.sleep(nanoseconds: 100_000_000)
            }
        }
        for attempt in 0..<5 {
            let axWindows = attribute(root, kAXWindowsAttribute) as? [AXUIElement] ?? []
            let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: true)
            try app.validate()
            let targets: [WindowTarget] = content.windows.compactMap { window in
                guard window.owningApplication?.processID == app.pid, window.windowLayer == 0,
                      window.frame.width > 20, window.frame.height > 20 else { return nil }
                let matches = axWindows.filter { element in
                    guard let bounds = frame(element) else { return false }
                    return abs(bounds.minX - window.frame.minX) < 2 && abs(bounds.minY - window.frame.minY) < 2
                        && abs(bounds.width - window.frame.width) < 2 && abs(bounds.height - window.frame.height) < 2
                }
                // Never guess a window from a repeated title or take the first match.
                guard matches.count == 1, let element = matches.first else { return nil }
                AXUIElementSetMessagingTimeout(element, 0.2)
                return WindowTarget(id: window.windowID, title: window.title ?? "Untitled window", element: element, frame: window.frame)
            }.sorted { $0.id < $1.id }
            if !targets.isEmpty { return targets }
            // WindowServer and Accessibility can report different frames during
            // opening animation. Refresh reads briefly; never guess or dispatch input.
            if attempt < 4 { try await Task.sleep(nanoseconds: 100_000_000) }
        }
        return []
    }

    func validate(_ target: WindowTarget, app: AppIdentity, requireFrontmost: Bool = false) throws -> CGRect {
        try requirePermissions()
        try app.validate()
        var pid: pid_t = 0
        guard AXUIElementGetPid(target.element, &pid) == .success, pid == app.pid,
              let bounds = frame(target.element), bounds.width > 20, bounds.height > 20,
              (attribute(target.element, kAXMinimizedAttribute) as? Bool) != true,
              let info = CGWindowListCopyWindowInfo(.optionIncludingWindow, target.id) as? [[String: Any]],
              info.contains(where: { ($0[kCGWindowOwnerPID as String] as? Int32) == app.pid
                  && ($0[kCGWindowIsOnscreen as String] as? Bool) == true }) else {
            throw UseError("window_unavailable", "The approved window is closed, minimized, or no longer on this desktop.", next: "human_takeover")
        }
        if requireFrontmost {
            let root = AXUIElementCreateApplication(app.pid)
            guard NSWorkspace.shared.frontmostApplication?.processIdentifier == app.pid,
                  let focused = elementAttribute(root, kAXFocusedWindowAttribute), CFEqual(focused, target.element) else {
                throw UseError("window_not_frontmost", "Choose Continue in the Computer Use controls to return to the approved window.", next: "human_takeover")
            }
        }
        return bounds
    }

    func read(_ target: WindowTarget) -> WindowState {
        var records: [ElementRecord] = []
        var protectedFrames: [CGRect] = []
        var visited: Set<AXUIElement> = []
        var truncated = false
        let started = ProcessInfo.processInfo.systemUptime
        func walk(_ element: AXUIElement, depth: Int) {
            guard depth < 24, visited.count < 1500, records.count < 300,
                  ProcessInfo.processInfo.systemUptime - started < 2 else { truncated = true; return }
            guard visited.insert(element).inserted else { return }
            let role = string(element, kAXRoleAttribute) ?? "AXUnknown"
            let secure = string(element, kAXSubroleAttribute) == "AXSecureTextField"
                || (attribute(element, "AXProtectedContent") as? Bool) == true
            let bounds = frame(element)
            if secure {
                if let bounds { protectedFrames.append(bounds) }
                // Do not read protected values, labels, descriptions, or descendants.
                return
            }
            let label = String((string(element, kAXTitleAttribute) ?? string(element, kAXDescriptionAttribute) ?? "").prefix(180))
            let value = string(element, kAXValueAttribute).map { String($0.prefix(500)) }
            var actionValues: CFArray?
            AXUIElementCopyActionNames(element, &actionValues)
            let actions = (actionValues as? [String] ?? []).filter { $0 == kAXPressAction }
            var settable = DarwinBoolean(false)
            AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable)
            let enabled = (attribute(element, kAXEnabledAttribute) as? Bool) != false
            if let bounds, bounds.width > 0, bounds.height > 0,
               !label.isEmpty || !(value ?? "").isEmpty || !actions.isEmpty || settable.boolValue {
                records.append(ElementRecord(ref: "e\(records.count + 1)", element: element, role: role,
                    label: label, value: value, frame: bounds, actions: actions,
                    settable: settable.boolValue, enabled: enabled, protected: false))
            }
            let children = attribute(element, kAXChildrenAttribute) as? [AXUIElement] ?? []
            for child in children { walk(child, depth: depth + 1) }
        }
        walk(target.element, depth: 0)
        return WindowState(records: records, visited: visited.count, truncated: truncated, protectedFrames: protectedFrames)
    }

    func validateRecord(_ record: ElementRecord, target: WindowTarget) throws {
        guard !record.protected, record.enabled,
              string(record.element, kAXSubroleAttribute) != "AXSecureTextField",
              (attribute(record.element, "AXProtectedContent") as? Bool) != true,
              let window = elementAttribute(record.element, kAXWindowAttribute), CFEqual(window, target.element),
              frame(record.element) == record.frame,
              string(record.element, kAXRoleAttribute) == record.role,
              String((string(record.element, kAXTitleAttribute) ?? string(record.element, kAXDescriptionAttribute) ?? "").prefix(180)) == record.label,
              string(record.element, kAXValueAttribute).map({ String($0.prefix(500)) }) == record.value,
              (attribute(record.element, kAXEnabledAttribute) as? Bool) != false else {
            throw UseError("stale_element", "The exact accessible element changed or is unavailable. Observe again.", next: "observe")
        }
    }

    func digest(_ state: WindowState) -> Data {
        let records: [[String: Any]] = state.records.map { record in
            ["role": record.role, "label": record.label, "value": record.value ?? NSNull(),
             "frame": [record.frame.minX, record.frame.minY, record.frame.width, record.frame.height],
             "enabled": record.enabled, "actions": record.actions, "settable": record.settable]
        }
        let secure = state.protectedFrames.map { [$0.minX, $0.minY, $0.width, $0.height] }
        let data = try! JSONSerialization.data(withJSONObject: ["records": records, "secure": secure, "truncated": state.truncated], options: [.sortedKeys])
        return Data(SHA256.hash(data: data))
    }
    func imageDigest(_ data: Data) -> Data { Data(SHA256.hash(data: data)) }

    func checkHit(_ point: CGPoint, target: WindowTarget, app: AppIdentity) throws {
        var hit: AXUIElement?
        guard AXUIElementCopyElementAtPosition(AXUIElementCreateSystemWide(), Float(point.x), Float(point.y), &hit) == .success,
              let hit else { throw UseError("unverified_target", "This position cannot be verified as part of the approved window.", next: "observe") }
        var pid: pid_t = 0
        let window = elementAttribute(hit, kAXWindowAttribute)
        guard AXUIElementGetPid(hit, &pid) == .success, pid == app.pid,
              CFEqual(hit, target.element) || (window.map { CFEqual($0, target.element) } == true),
              string(hit, kAXSubroleAttribute) != "AXSecureTextField",
              (attribute(hit, "AXProtectedContent") as? Bool) != true else {
            throw UseError("outside_window", "The position is covered by another window or a protected field.", next: "human_takeover")
        }
    }

    func checkFocusedField(target: WindowTarget, app: AppIdentity) throws {
        let root = AXUIElementCreateApplication(app.pid)
        guard let focused = elementAttribute(root, kAXFocusedUIElementAttribute),
              let window = elementAttribute(focused, kAXWindowAttribute), CFEqual(window, target.element),
              string(focused, kAXSubroleAttribute) != "AXSecureTextField",
              (attribute(focused, "AXProtectedContent") as? Bool) != true else {
            throw UseError("protected_input", "Focus a normal control in the approved window. Enter passwords yourself.", next: "human_takeover")
        }
    }

    func capture(target: WindowTarget, app: AppIdentity, bounds: CGRect, state: WindowState) async throws -> (Data, Int, Int) {
        let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: true)
        guard let window = content.windows.first(where: { $0.windowID == target.id && $0.owningApplication?.processID == app.pid }),
              window.frame == bounds else { throw UseError("stale_observation", "The selected window moved before capture.", next: "observe") }
        let config = SCStreamConfiguration()
        let ratio = min(1, 1600 / max(bounds.width, bounds.height))
        config.width = max(1, Int(bounds.width * ratio))
        config.height = max(1, Int(bounds.height * ratio))
        config.showsCursor = false
        config.ignoreShadowsSingleWindow = true
        let image = try await SCScreenshotManager.captureImage(contentFilter: SCContentFilter(desktopIndependentWindow: window), configuration: config)
        // Capture only the window; a failure never broadens into a display capture.
        guard let context = CGContext(data: nil, width: image.width, height: image.height, bitsPerComponent: 8,
            bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else {
            throw UseError("capture_failed", "Could not encode this window.", next: "observe")
        }
        context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
        context.setFillColor(CGColor(gray: 0.15, alpha: 1))
        for rect in state.protectedFrames {
            let clipped = rect.intersection(bounds)
            guard !clipped.isNull, !clipped.isEmpty else { continue }
            let xScale = CGFloat(image.width) / bounds.width
            let yScale = CGFloat(image.height) / bounds.height
            context.fill(CGRect(x: (clipped.minX - bounds.minX) * xScale,
                y: CGFloat(image.height) - (clipped.maxY - bounds.minY) * yScale,
                width: clipped.width * xScale, height: clipped.height * yScale).insetBy(dx: -2, dy: -2))
        }
        guard let redacted = context.makeImage(), let data = NSBitmapImageRep(cgImage: redacted).representation(using: .png, properties: [:]) else {
            throw UseError("capture_failed", "Could not encode this window.", next: "observe")
        }
        return (data, image.width, image.height)
    }

    func requirePermissions() throws {
        guard AXIsProcessTrusted(), CGPreflightScreenCaptureAccess() else {
            throw UseError("permissions_required", "Open Computer Use settings and grant Accessibility and Screen Recording yourself.", next: "human_takeover")
        }
    }
    func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
        var value: CFTypeRef?
        let result = AXUIElementCopyAttributeValue(element, name as CFString, &value)
        return result == .success ? value : nil
    }
    func elementAttribute(_ element: AXUIElement, _ name: String) -> AXUIElement? {
        guard let value = attribute(element, name), CFGetTypeID(value) == AXUIElementGetTypeID() else { return nil }
        return (value as! AXUIElement) // Core Foundation type ID checked above.
    }
    func string(_ element: AXUIElement, _ name: String) -> String? {
        let value = attribute(element, name)
        if let string = value as? String { return string }
        if let number = value as? NSNumber { return number.stringValue }
        return nil
    }
    func frame(_ element: AXUIElement) -> CGRect? {
        guard let rawPosition = attribute(element, kAXPositionAttribute), CFGetTypeID(rawPosition) == AXValueGetTypeID(),
              let rawSize = attribute(element, kAXSizeAttribute), CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil }
        let position = rawPosition as! AXValue
        let size = rawSize as! AXValue
        guard AXValueGetType(position) == .cgPoint, AXValueGetType(size) == .cgSize else { return nil }
        var point = CGPoint.zero; var dimensions = CGSize.zero
        guard AXValueGetValue(position, .cgPoint, &point), AXValueGetValue(size, .cgSize, &dimensions) else { return nil }
        return CGRect(origin: point, size: dimensions)
    }
}