Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
19.6 KB
Raw Blame
import AppKit
import Foundation

actor ComputerUseRuntime {
    private let accessibility = AccessibilityService()
    private let foregroundInput = InputService()
    private var lastSnapshot: AppSnapshot?
    private var strictMode = true
    private var activationSession: BackgroundActivationSession?
    private var activationKey: String?
    private var activatedWindowKey: String?
    private var activationPreviousPID: pid_t?
    private var activationTargetPID: pid_t?
    private var frontmostMonitor: FrontmostApplicationMonitor?
    private var snapshotSequence = 0
    private var previousLabelsByWindow: [String: Set<String>] = [:]
    private var recentActions: [String] = []
    private var staleSnapshotReason: String?

    func setStrictMode(_ enabled: Bool) -> ActionMetadata {
        strictMode = enabled
        if !enabled {
            resetBackgroundActivation()
        }
        return ActionMetadata(
            ok: true,
            path: .none,
            strictMode: enabled,
            backgroundSafe: enabled,
            fallbackUsed: false,
            message: enabled ? "Strict background mode enabled." : "Strict background mode disabled. Foreground fallback is allowed."
        )
    }

    func snapshot(appName: String?, pid: Int? = nil, windowTitle: String? = nil, strict requestedStrict: Bool?) async throws -> AppSnapshot {
        let effectiveStrict = requestedStrict ?? strictMode
        if !effectiveStrict {
            resetBackgroundActivation()
        }

        let requestedPID = pid.map(pid_t.init)
        var target = try accessibility.resolveTarget(appName: appName, pid: requestedPID, windowTitle: windowTitle)
        let backgroundActivated: Bool
        if effectiveStrict, !target.isFrontmost {
            backgroundActivated = try await ensureBackgroundActivation(target: target)
            target = try accessibility.resolveTarget(appName: appName, pid: requestedPID, windowTitle: windowTitle)
        } else {
            backgroundActivated = false
        }

        let snapshot = try await accessibility.snapshot(
            target: target,
            strictMode: effectiveStrict,
            backgroundActivated: backgroundActivated
        )
        let annotatedSnapshot = annotate(snapshot: snapshot)
        staleSnapshotReason = nil
        lastSnapshot = annotatedSnapshot
        return annotatedSnapshot
    }

    func click(snapshotID: String?, ref: String?, index: Int?, imageX: Double?, imageY: Double?, screenX: Double? = nil, screenY: Double? = nil, clickCount: Int, strict requestedStrict: Bool?) async throws -> ActionMetadata {
        let snapshot = try requireSnapshot(snapshotID: snapshotID)
        let effectiveStrict = requestedStrict ?? snapshot.strictMode
        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)

        if let record = findRecord(ref: ref, index: index, in: snapshot) {
            let fresh = freshRecord(matching: record, in: snapshot)
            if let fresh {
                AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
                if fresh.semantic.capabilities.canPress, accessibility.press(record: fresh) {
                    return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: effectiveStrict, backgroundSafe: true, fallbackUsed: false, message: "Pressed \(fresh.semantic.ref) via AXPress."))
                }
                if fresh.semantic.capabilities.canFocus, accessibility.focus(record: fresh) {
                    return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: effectiveStrict, backgroundSafe: true, fallbackUsed: false, message: "Focused \(fresh.semantic.ref) via AX."))
                }
                return try await clickPoint(fresh.semantic.frame.center, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: true)
            }
            return try await clickPoint(record.semantic.frame.center, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: true)
        }

        if let screenX, let screenY {
            return try await clickPoint(CGPoint(x: screenX, y: screenY), clickCount: clickCount, strict: effectiveStrict, fallbackUsed: false)
        }

        if let imageX, let imageY {
            let point = snapshot.screenshotMeta.toScreen(imageX: imageX, imageY: imageY)
            return try await clickPoint(point, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: false)
        }

        throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
    }

    func typeText(snapshotID: String?, text: String, strict requestedStrict: Bool?) throws -> ActionMetadata {
        let snapshot = try requireSnapshot(snapshotID: snapshotID)
        let effectiveStrict = requestedStrict ?? snapshot.strictMode
        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
        if effectiveStrict {
            try BackgroundInputDispatcher.typeText(pid: snapshot.pid, text: text)
            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Typed text with postToPid."))
        }

        try foregroundInput.typeText(text)
        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Typed text with foreground HID fallback."))
    }

    func pressKey(snapshotID: String?, combo: String, strict requestedStrict: Bool?) throws -> ActionMetadata {
        let snapshot = try requireSnapshot(snapshotID: snapshotID)
        let effectiveStrict = requestedStrict ?? snapshot.strictMode
        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
        if effectiveStrict {
            try BackgroundInputDispatcher.pressKey(pid: snapshot.pid, combo: combo)
            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Pressed key with postToPid."))
        }

        try foregroundInput.pressKey(combo)
        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Pressed key with foreground HID fallback."))
    }

    func scroll(snapshotID: String?, direction: String?, pages: Double, imageX: Double?, imageY: Double?, strict requestedStrict: Bool?) throws -> ActionMetadata {
        let snapshot = try requireSnapshot(snapshotID: snapshotID)
        let effectiveStrict = requestedStrict ?? snapshot.strictMode
        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
        let amount = max(1, Int32(pages * 5))
        let deltas = scrollDeltas(direction: direction, amount: amount)
        let point: CGPoint = {
            if let imageX, let imageY {
                return snapshot.screenshotMeta.toScreen(imageX: imageX, imageY: imageY)
            }
            return CGPoint(x: snapshot.screenshotMeta.capturedBounds.midX, y: snapshot.screenshotMeta.capturedBounds.midY)
        }()
        AgentCursorOverlay.shared.show(at: point)

        if effectiveStrict {
            guard let windowNumber = snapshot.windowNumber else {
                throw ComputerUseError.strictModeViolation("background scroll requires a CG window number")
            }
            try BackgroundInputDispatcher.scroll(pid: snapshot.pid, windowNumber: windowNumber, point: point, deltaX: deltas.x, deltaY: deltas.y)
            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Scrolled with postToPid."))
        }

        try foregroundInput.scroll(point: point, deltaX: deltas.x, deltaY: deltas.y)
        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Scrolled with foreground HID fallback."))
    }

    func setValue(snapshotID: String?, ref: String?, index: Int?, value: String) throws -> ActionMetadata {
        let snapshot = try requireSnapshot(snapshotID: snapshotID)
        try validateSnapshotForAction(snapshot: snapshot, strict: snapshot.strictMode)
        guard let record = findRecord(ref: ref, index: index, in: snapshot) else {
            throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
        }
        guard let fresh = freshRecord(matching: record, in: snapshot) else {
            throw ComputerUseError.staleSnapshot("The target element changed. Take a new snapshot before setting its value.")
        }
        AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
        let axWriteAccepted = accessibility.setValue(record: fresh, value: value)
        if axWriteAccepted, valueMatches(record: fresh, expected: value) {
            return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: snapshot.strictMode, backgroundSafe: true, fallbackUsed: false, message: "Set \(fresh.semantic.ref) via AXValue."))
        }

        // Web inputs (Chromium/Electron) accept AXValue writes without applying
        // them. Fall back to focus + select-all + retype, then verify.
        _ = accessibility.focus(record: fresh)
        try BackgroundInputDispatcher.pressKey(pid: snapshot.pid, combo: "command+a")
        try BackgroundInputDispatcher.typeText(pid: snapshot.pid, text: value)
        Thread.sleep(forTimeInterval: 0.08)
        let verified = valueMatches(record: fresh, expected: value)
        return recordAction(ActionMetadata(
            ok: verified,
            path: .backgroundCGEvent,
            strictMode: snapshot.strictMode,
            backgroundSafe: true,
            fallbackUsed: true,
            message: verified
                ? "AXValue write did not stick; set \(fresh.semantic.ref) via select-all + retype fallback."
                : "Could not verify the value of \(fresh.semantic.ref) after AXValue and retype fallback."
        ))
    }

    private func valueMatches(record: AXElementRecord, expected: String) -> Bool {
        accessibility.value(record: record) == expected
    }

    func performAction(snapshotID: String?, ref: String?, index: Int?, action: String) throws -> ActionMetadata {
        let snapshot = try requireSnapshot(snapshotID: snapshotID)
        try validateSnapshotForAction(snapshot: snapshot, strict: snapshot.strictMode)
        guard let record = findRecord(ref: ref, index: index, in: snapshot) else {
            throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
        }
        guard let fresh = freshRecord(matching: record, in: snapshot) else {
            throw ComputerUseError.staleSnapshot("The target element changed. Take a new snapshot before performing another action.")
        }
        AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
        let ok = accessibility.performAction(record: fresh, action: action)
        return recordAction(ActionMetadata(ok: ok, path: .accessibility, strictMode: snapshot.strictMode, backgroundSafe: true, fallbackUsed: false, message: ok ? "Performed \(action) on \(fresh.semantic.ref)." : "AX action \(action) failed."))
    }

    func wait(milliseconds: Int) async -> ActionMetadata {
        let clamped = max(0, min(milliseconds, 10_000))
        try? await Task.sleep(nanoseconds: UInt64(clamped) * 1_000_000)
        return ActionMetadata(ok: true, path: .none, strictMode: strictMode, backgroundSafe: true, fallbackUsed: false, message: "Waited \(clamped)ms.")
    }

    private func clickPoint(_ point: CGPoint, clickCount: Int, strict: Bool, fallbackUsed: Bool) async throws -> ActionMetadata {
        let snapshot = try requireSnapshot()
        try validateSnapshotForAction(snapshot: snapshot, strict: strict)
        AgentCursorOverlay.shared.show(at: point)
        if strict {
            guard let windowNumber = snapshot.windowNumber else {
                throw ComputerUseError.strictModeViolation("background click requires a CG window number")
            }
            try await BackgroundInputDispatcher.click(pid: snapshot.pid, windowNumber: windowNumber, point: point, doubleClick: clickCount >= 2)
            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: fallbackUsed, message: "Clicked with postToPid at \(Int(point.x)),\(Int(point.y))."))
        }

        try await foregroundInput.click(point: point, doubleClick: clickCount >= 2)
        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Clicked with foreground HID fallback at \(Int(point.x)),\(Int(point.y))."))
    }

    private func ensureBackgroundActivation(target: WindowTarget) async throws -> Bool {
        ensureFrontmostMonitor()
        guard let previousPID = NSWorkspace.shared.frontmostApplication?.processIdentifier else {
            throw ComputerUseError.noFrontmostApplication
        }
        let nextActivationKey = "\(previousPID):\(target.pid)"
        if activationKey != nextActivationKey {
            resetBackgroundActivation()
            let next = BackgroundActivationSession(previousPID: previousPID, targetPID: target.pid)
            try next.start()
            activationSession = next
            activationKey = nextActivationKey
            activationPreviousPID = previousPID
            activationTargetPID = target.pid
            activatedWindowKey = nil
        }

        let nextWindowKey = "\(target.pid):\(target.windowNumber ?? -1)"
        if activatedWindowKey != nextWindowKey {
            guard let activationSession else {
                throw ComputerUseError.strictModeViolation("background activation session was not created")
            }
            try await activationSession.activate(target: target)
            activatedWindowKey = nextWindowKey
        }
        return true
    }

    private func ensureFrontmostMonitor() {
        if frontmostMonitor != nil { return }
        frontmostMonitor = FrontmostApplicationMonitor { [weak self] pid in
            guard let runtime = self else { return }
            Task { await runtime.frontmostApplicationChanged(pid: pid) }
        }
    }

    private func frontmostApplicationChanged(pid: pid_t?) {
        guard let pid, activationSession != nil else { return }
        if pid == activationPreviousPID {
            staleSnapshotReason = "The user returned focus to the previous app. Take a new snapshot before sending more input."
        } else if pid == activationTargetPID {
            staleSnapshotReason = "The target app became frontmost. Take a new snapshot before continuing."
        } else {
            staleSnapshotReason = "The user changed app focus. Take a new snapshot before sending more input."
        }
        resetBackgroundActivation()
    }

    private func resetBackgroundActivation() {
        activationSession?.stop()
        activationSession = nil
        activationKey = nil
        activatedWindowKey = nil
        activationPreviousPID = nil
        activationTargetPID = nil
    }

    private func requireSnapshot(snapshotID: String? = nil) throws -> AppSnapshot {
        guard let lastSnapshot else { throw ComputerUseError.noSnapshot }
        if let snapshotID, snapshotID != lastSnapshot.id {
            throw ComputerUseError.staleSnapshot("Snapshot \(snapshotID) is no longer current. Take a new snapshot before retrying.")
        }
        return lastSnapshot
    }

    private func validateSnapshotForAction(snapshot: AppSnapshot, strict: Bool) throws {
        if let staleSnapshotReason {
            throw ComputerUseError.staleSnapshot(staleSnapshotReason)
        }
        guard strict else { return }
        if NSWorkspace.shared.frontmostApplication?.processIdentifier == snapshot.pid { return }
        guard activationSession != nil, activationTargetPID == snapshot.pid else {
            throw ComputerUseError.staleSnapshot("The target app is no longer safely activated. Take a new snapshot before retrying.")
        }
    }

    private func annotate(snapshot: AppSnapshot) -> AppSnapshot {
        snapshotSequence += 1
        let key = windowKey(snapshot: snapshot)
        let labels = Set(snapshot.records.map { stableLabel(for: $0.semantic) }.filter { !$0.isEmpty })
        let previous = previousLabelsByWindow[key] ?? []
        previousLabelsByWindow[key] = labels

        let added = labels.subtracting(previous).sorted()
        let removed = previous.subtracting(labels).sorted()
        return snapshot.withSessionMetadata(
            id: UUID().uuidString.lowercased(),
            observation: snapshotSequence,
            recentActions: Array(recentActions.suffix(8)),
            addedLabels: Array(added.prefix(16)),
            removedLabels: Array(removed.prefix(16))
        )
    }

    private func recordAction(_ metadata: ActionMetadata) -> ActionMetadata {
        recentActions.append(metadata.message)
        if recentActions.count > 12 {
            recentActions.removeFirst(recentActions.count - 12)
        }
        return metadata
    }

    private func freshRecord(matching record: AXElementRecord, in snapshot: AppSnapshot) -> AXElementRecord? {
        guard let target = try? accessibility.resolveTarget(appName: snapshot.appName, pid: snapshot.pid, windowTitle: snapshot.windowTitle) else {
            return nil
        }
        let records = accessibility.records(target: target)
        if let index = records.firstIndex(where: { $0.semantic.id == record.semantic.id }), stableMatch(records[index].semantic, record.semantic) {
            return records[index]
        }
        return records.first { stableMatch($0.semantic, record.semantic) }
    }

    private func stableMatch(_ candidate: SemanticAXElement, _ target: SemanticAXElement) -> Bool {
        candidate.role == target.role && stableLabel(for: candidate) == stableLabel(for: target)
    }

    private func stableLabel(for element: SemanticAXElement) -> String {
        let value = element.value?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
        let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
        let display = value.isEmpty ? label : "\(label) \(value)"
        return "\(element.role):\(display)"
            .components(separatedBy: .whitespacesAndNewlines)
            .filter { !$0.isEmpty }
            .joined(separator: " ")
            .lowercased()
    }

    private func windowKey(snapshot: AppSnapshot) -> String {
        "\(snapshot.pid):\(snapshot.windowNumber ?? -1)"
    }

    private func findRecord(ref: String?, index: Int?, in snapshot: AppSnapshot) -> AXElementRecord? {
        if let ref {
            let normalized = normalizeRef(ref)
            return snapshot.records.first { $0.semantic.ref == normalized }
        }
        if let index {
            if let byID = snapshot.records.first(where: { $0.semantic.id == index }) {
                return byID
            }
            if index >= 0 && index < snapshot.records.count {
                return snapshot.records[index]
            }
        }
        return nil
    }

    private func normalizeRef(_ raw: String) -> String {
        let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
        if trimmed.hasPrefix("{e"), trimmed.hasSuffix("}") { return trimmed }
        if trimmed.hasPrefix("e") { return "{\(trimmed)}" }
        return "{e\(trimmed)}"
    }

    private func scrollDeltas(direction: String?, amount: Int32) -> (x: Int32, y: Int32) {
        switch direction?.lowercased() ?? "down" {
        case "up": return (0, amount)
        case "left": return (amount, 0)
        case "right": return (-amount, 0)
        default: return (0, -amount)
        }
    }
}