Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
9.1 KB
Raw Blame
import AppKit
import ApplicationServices

@MainActor
final class MacInput {
    typealias SetWindowLocation = @convention(c) (CGEvent, CGPoint) -> Void
    // AppKit-targeted events carry a second, window-local point. On recent
    // macOS, setting only the public screen location leaves that point invalid.
    // Keep this compatibility hook isolated and fail before input if unavailable.
    static let setWindowLocation: SetWindowLocation? = {
        guard let library = dlopen(nil, RTLD_LAZY) else { return nil }
        defer { dlclose(library) }
        guard let symbol = dlsym(library, "CGEventSetWindowLocation") else { return nil }
        return unsafeBitCast(symbol, to: SetWindowLocation.self)
    }()
    private var heldPointer: (pid: pid_t, point: CGPoint, windowID: CGWindowID, frame: CGRect)?
    func releaseAll() {
        if let heldPointer { try? mouse(.leftMouseUp, heldPointer.point, heldPointer.pid, 1, heldPointer.windowID, heldPointer.frame) }
        heldPointer = nil
    }
    // The input backend accepts a pinned process, never a frontmost-app default.
    // Every individual event rechecks the live session, including drag and Unicode typing.
    func execute(_ action: Action, app: AppIdentity, target: WindowTarget, lease: ObservationLease,
                 access: MacAccessibility, records: [ElementRecord], check: () throws -> Void) async throws -> String {
        try check()
        switch action {
        case .press(let ref), .setValue(let ref, _):
            guard let record = records.first(where: { $0.ref == ref }) else { throw UseError("unknown_ref", "Use an element ref from the current observation.", next: "observe") }
            try access.validateRecord(record, target: target)
            if case .setValue(_, let text) = action {
                var settable = DarwinBoolean(false)
                guard record.settable,
                      AXUIElementIsAttributeSettable(record.element, kAXValueAttribute as CFString, &settable) == .success, settable.boolValue else {
                    throw UseError("unsupported_action", "This control does not support setting a value. Choose another explicit action.", next: "observe")
                }
                guard AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, text as CFString) == .success else {
                    throw UseError("action_failed", "The app did not accept the value. Observe before retrying.", next: "observe")
                }
            } else {
                guard record.actions.contains(kAXPressAction) else { throw UseError("unsupported_action", "This control has no accessible press action. Use an explicitly approved visual session.", next: "open_session") }
                guard AXUIElementPerformAction(record.element, kAXPressAction as CFString) == .success else {
                    throw UseError("action_failed", "The app did not accept the press. Observe before retrying.", next: "observe")
                }
            }
            return "accessibility"
        case .click(let point, let count):
            let screen = try lease.screenPoint(point)
            try access.checkHit(screen, target: target, app: app)
            for click in 1...count {
                try check()
                try mouse(.leftMouseDown, screen, app.pid, click, target.id, lease.frame)
                // Always release a dispatched press, including cancellation.
                try mouse(.leftMouseUp, screen, app.pid, click, target.id, lease.frame)
                if click < count { try await Task.sleep(nanoseconds: 70_000_000) }
            }
        case .scroll(let point, let dx, let dy):
            let screen = try lease.screenPoint(point)
            try access.checkHit(screen, target: target, app: app)
            guard let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: dy, wheel2: dx, wheel3: 0) else {
                throw UseError("input_failed", "Could not create a scroll event.")
            }
            event.location = screen
            event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(target.id))
            event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(target.id))
            post(event, app.pid)
        case .key(let key):
            try access.checkFocusedField(target: target, app: app)
            let code: CGKeyCode
            let flags: CGEventFlags
            switch key {
            case "select_all": code = 0; flags = .maskCommand
            case "undo": code = 6; flags = .maskCommand
            case "redo": code = 6; flags = [.maskCommand, .maskShift]
            default:
                guard let mapped = NativeKey.codes[key] else { throw UseError("unsupported_key", "Unknown key.") }
                code = mapped; flags = []
            }
            try keyboard(code, flags: flags, text: nil, pid: app.pid)
        case .type(let text):
            try access.checkFocusedField(target: target, app: app)
            // Chunk by grapheme; never split UTF-16 surrogate pairs or use the shared clipboard.
            for character in text {
                try check()
                try access.checkFocusedField(target: target, app: app)
                try keyboard(0, flags: [], text: String(character), pid: app.pid)
                await Task.yield()
            }
        case .drag(let points):
            let path = try points.map(lease.screenPoint)
            // Validate all waypoints before pressing; recheck hit testing while dragging.
            for point in path { try access.checkHit(point, target: target, app: app) }
            guard let first = path.first else { throw UseError("invalid_arguments", "Drag path is empty.") }
            try mouse(.leftMouseDown, first, app.pid, 1, target.id, lease.frame)
            defer { releaseAll() }
            for point in path.dropFirst() {
                try await Task.sleep(nanoseconds: 20_000_000)
                try check()
                try access.checkHit(point, target: target, app: app)
                try mouse(.leftMouseDragged, point, app.pid, 1, target.id, lease.frame)
            }
        }
        return "targeted_input"
    }
    private func post(_ event: CGEvent, _ pid: pid_t) {
        event.setIntegerValueField(.eventSourceUnixProcessID, value: Int64(ProcessInfo.processInfo.processIdentifier))
        event.postToPid(pid)
    }
    private func mouse(_ type: CGEventType, _ point: CGPoint, _ pid: pid_t, _ count: Int, _ windowID: CGWindowID, _ frame: CGRect) throws {
        guard let setWindowLocation = Self.setWindowLocation else {
            throw UseError("unsupported_pointer_input", "This macOS version does not support scoped pointer input. Use accessible app controls instead.", next: "human_takeover")
        }
        let eventType: NSEvent.EventType
        switch type {
        case .leftMouseDown: eventType = .leftMouseDown
        case .leftMouseUp: eventType = .leftMouseUp
        case .leftMouseDragged: eventType = .leftMouseDragged
        default: throw UseError("input_failed", "Unsupported pointer event.")
        }
        let location = NSPoint(x: point.x - frame.minX, y: frame.maxY - point.y)
        guard let native = NSEvent.mouseEvent(with: eventType, location: location, modifierFlags: [],
                timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: Int(windowID), context: nil,
                eventNumber: 0, clickCount: count, pressure: type == .leftMouseUp ? 0 : 1),
              let event = native.cgEvent?.copy() else {
            throw UseError("input_failed", "Could not create a window-targeted mouse event.")
        }
        event.location = point
        setWindowLocation(event, CGPoint(x: point.x - frame.minX, y: point.y - frame.minY))
        event.setIntegerValueField(.mouseEventButtonNumber, value: 0)
        event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(windowID))
        event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(windowID))
        event.setIntegerValueField(.mouseEventClickState, value: Int64(count))
        post(event, pid)
        if type == .leftMouseDown || type == .leftMouseDragged { heldPointer = (pid, point, windowID, frame) }
        if type == .leftMouseUp { heldPointer = nil }
    }
    private func keyboard(_ code: CGKeyCode, flags: CGEventFlags, text: String?, pid: pid_t) throws {
        guard let down = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
              let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false) else {
            throw UseError("input_failed", "Could not create a keyboard event.")
        }
        down.flags = flags; up.flags = flags
        if let text {
            let units = Array(text.utf16)
            units.withUnsafeBufferPointer { buffer in
                if let base = buffer.baseAddress {
                    down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: base)
                    up.keyboardSetUnicodeString(stringLength: units.count, unicodeString: base)
                }
            }
        }
        post(down, pid); post(up, pid)
    }
}