Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
37.9 KB
Raw Blame
import AppKit
import ApplicationServices
import CryptoKit
import ScreenCaptureKit
import Vision

// `ComputerUse worker`: the macOS native layer of the desktop's shared
// computer-use host (apps/server/src/computer-use/shared/portable-host.mjs,
// contract v2). Same operations and shapes as shared/native-worker.mjs
// (Windows/X11): list, window, screens, cursor, focus, state, capture, hit,
// act, start, watch, stop, input-source, shutdown; events ready, held,
// person-input. The tool surface, results, approvals and media spool live in
// the shared host; this process only reads windows and posts input.
//
// Transport: stdin JSON lines {id, method, params}; stdout JSON lines
// {id, result} | {id, error: {code, message}} | {event, ...}. A capture
// result header carries result.bytes and is followed by that many raw RGBA
// bytes (WorkerFrame).
//
// Coordinates: global screen points, origin top-left of the primary display
// (the CGWindow/CGEvent space). Captures are native pixels (points × backing
// scale, capped by max_long_side); secure/caret/text boxes are in pixels of
// the capture.

struct WorkerWindow {
    let id: CGWindowID
    let pid: pid_t
    let identity: String
    let executable: String
    let appId: String
    let appName: String
    let title: String
    let bounds: CGRect
    let scale: CGFloat
    let display: CGDirectDisplayID

    var json: [String: Any] {
        ["id": Int(id), "pid": Int(pid), "identity": identity, "executable": executable, "appId": appId, "appName": appName,
         "title": title, "bounds": ["x": Int(bounds.minX), "y": Int(bounds.minY), "width": Int(bounds.width), "height": Int(bounds.height)],
         "dpi": Int((72 * scale).rounded()), "scale": Double(scale), "monitor": String(display)]
    }
}

private struct HeldPointer {
    let pid: pid_t
    let point: CGPoint
    let window: CGWindowID
    let frame: CGRect
    let flags: CGEventFlags
}

@MainActor
final class NativeWorker {
    private var generation = 0
    private var busy = false
    private var interrupting = false
    private var monitors: [Any] = []
    private var observers: [NSObjectProtocol] = []
    private var anchor: CGPoint?
    private var lastInput: [String: Any] = ["synthetic": false, "at": 0, "kind": ""]
    private var held: HeldPointer?
    private var scCache: (at: TimeInterval, windows: [CGWindowID: SCWindow])?
    private var ocrCache: (digest: Data, lines: [[String: Any]]?)?
    private let ownPid = ProcessInfo.processInfo.processIdentifier

    // MARK: transport

    func start() {
        let ax = AXIsProcessTrusted(), capture = CGPreflightScreenCaptureAccess()
        if ax && capture { send(["event": "ready", "supported": true, "platform": "darwin"]) }
        else {
            send(["event": "ready", "supported": false, "platform": "darwin", "code": "permissions_required",
                  "error": "Grant Accessibility and Screen Recording to OmniRush.ai in System Settings, then restart computer use."])
        }
        let center = NSWorkspace.shared.notificationCenter
        for name in [NSWorkspace.willSleepNotification, NSWorkspace.screensDidSleepNotification, NSWorkspace.sessionDidResignActiveNotification] {
            observers.append(center.addObserver(forName: name, object: nil, queue: .main) { [weak self] _ in
                MainActor.assumeIsolated { self?.personInput(force: true) }
            })
        }
    }

    private func send(_ message: [String: Any], binary: Data? = nil) {
        guard let data = WorkerFrame.encode(message, binary: binary) else { return }
        do { try FileHandle.standardOutput.write(contentsOf: data) } catch { exit(0) }
    }

    func receive(_ bytes: Data) async {
        guard let object = try? JSONSerialization.jsonObject(with: bytes) as? [String: Any],
              let method = object["method"] as? String else { return }
        let id = object["id"] ?? NSNull()
        let params = object["params"] as? [String: Any] ?? [:]
        var owns = false
        do {
            switch method {
            case "shutdown":
                generation += 1; releaseHeld(); stopMonitoring()
                while busy { try? await Task.sleep(nanoseconds: 10_000_000) }
                send(["id": id, "result": [:]])
                exit(0)
            case "start", "watch":
                guard !busy else { throw UseError("busy", "Desktop operations must be sequential.") }
                startMonitoring(interrupt: method == "start")
                send(["id": id, "result": ["generation": generation]]); return
            case "stop":
                generation += 1; releaseHeld(); stopMonitoring()
                send(["id": id, "result": [:]]); return
            case "input-source":
                send(["id": id, "result": lastInput]); return
            default: break
            }
            guard !busy else { throw UseError("busy", "Desktop operations must be sequential.") }
            busy = true; owns = true
            defer { if owns { busy = false } }
            switch method {
            case "list": send(["id": id, "result": list().map(\.json)])
            case "window": send(["id": id, "result": try window(Self.windowID(params["id"])).json])
            case "screens": send(["id": id, "result": screens()])
            case "cursor":
                if let p = CGEvent(source: nil)?.location { send(["id": id, "result": ["x": Int(p.x.rounded()), "y": Int(p.y.rounded())]]) }
                else { send(["id": id, "result": NSNull()]) }
            case "state": send(["id": id, "result": ["generation": generation, "available": available()]])
            case "focus":
                try await focus(try expected(params["window"]))
                send(["id": id, "result": ["generation": generation]])
            case "hit":
                let w = try expected(params["window"])
                let points = (params["points"] as? [[String: Any]] ?? []).map { CGPoint(x: Self.number($0["x"]) ?? -1e9, y: Self.number($0["y"]) ?? -1e9) }
                send(["id": id, "result": ["inside": points.map { hit(w, $0) }]])
            case "check":
                _ = try check(try expected(params["window"]), generation: Self.int(params["generation"]) ?? -1, deadline: Self.number(params["deadline"]) ?? 0)
                send(["id": id, "result": [:]])
            case "capture":
                let maxLongSide = Self.int(params["max_long_side"]) ?? 3840
                let (header, pixels) = try await capture(try expected(params["window"]), maxLongSide: max(320, min(7680, maxLongSide)))
                send(["id": id, "result": header], binary: pixels)
            case "act":
                guard let action = params["action"] as? [String: Any] else { throw UseError("invalid_action", "An action object is required.") }
                try await act(try expected(params["window"]), generation: Self.int(params["generation"]) ?? -1, action: action, deadline: Self.number(params["deadline"]) ?? 0)
                send(["id": id, "result": ["generation": generation]])
            default: throw UseError("unknown_method", "Unknown native operation.")
            }
        } catch {
            let failure = (error as? UseError) ?? UseError("native_error", error.localizedDescription)
            send(["id": id, "error": ["code": failure.code, "message": failure.message]])
        }
    }

    // MARK: parameters

    static func number(_ value: Any?) -> Double? {
        guard let n = value as? NSNumber, CFGetTypeID(n) != CFBooleanGetTypeID(), n.doubleValue.isFinite else { return nil }
        return n.doubleValue
    }
    static func int(_ value: Any?) -> Int? { number(value).flatMap { $0.rounded() == $0 && abs($0) < 9e15 ? Int($0) : nil } }
    static func windowID(_ value: Any?) throws -> CGWindowID {
        if let n = int(value), n > 0, n <= Int(UInt32.max) { return CGWindowID(n) }
        if let s = value as? String, let n = UInt32(s), n > 0 { return n }
        throw UseError("window_unavailable", "That window is not open.")
    }
    /// The window the shared host passed back ({id, identity, bounds}), checked against the live one.
    private func expected(_ value: Any?) throws -> WorkerWindow {
        guard let w = value as? [String: Any], let identity = w["identity"] as? String else { throw UseError("window_unavailable", "That window is not open.") }
        let current = try window(try Self.windowID(w["id"]))
        guard current.identity == identity else { throw UseError("window_changed", "The window changed. Observe again.") }
        return current
    }

    // MARK: windows

    private func appInfo(_ pid: pid_t) -> (NSRunningApplication, String, String, String)? {
        guard pid != ownPid, let app = NSRunningApplication(processIdentifier: pid), !app.isTerminated,
              app.activationPolicy == .regular, AppIdentity.isAllowed(app) || app.bundleIdentifier == nil,
              let exe = app.executableURL?.path else { return nil }
        return (app, exe, app.bundleIdentifier ?? exe, app.localizedName ?? (exe as NSString).lastPathComponent)
    }

    private func describe(_ info: [String: Any]) -> WorkerWindow? {
        guard let number = info[kCGWindowNumber as String] as? NSNumber,
              let pid = (info[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value,
              (info[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
              ((info[kCGWindowAlpha as String] as? NSNumber)?.doubleValue ?? 1) > 0.01,
              let rawBounds = info[kCGWindowBounds as String] as? NSDictionary,
              let bounds = CGRect(dictionaryRepresentation: rawBounds as CFDictionary),
              bounds.width > 20, bounds.height > 20,
              let app = appInfo(pid), let start = AppIdentity.ProcessStart(pid: pid) else { return nil }
        let (_, exe, appId, appName) = app
        let center = CGPoint(x: bounds.midX, y: bounds.midY)
        let screen = screenContaining(center)
        let display = (screen?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value ?? CGMainDisplayID()
        return WorkerWindow(id: number.uint32Value, pid: pid, identity: "\(pid):\(start.seconds).\(start.microseconds):\(number.uint32Value)",
            executable: exe, appId: appId, appName: appName, title: info[kCGWindowName as String] as? String ?? "",
            bounds: bounds.integral, scale: screen?.backingScaleFactor ?? 2, display: display)
    }

    func list() -> [WorkerWindow] {
        let infos = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
        return infos.compactMap(describe)
    }

    func window(_ id: CGWindowID) throws -> WorkerWindow {
        guard available(),
              let info = (CGWindowListCopyWindowInfo(.optionIncludingWindow, id) as? [[String: Any]])?.first,
              (info[kCGWindowIsOnscreen as String] as? Bool) == true,
              let w = describe(info), w.id == id else {
            throw UseError("window_unavailable", "Keep the approved app's window open and visible on an unlocked desktop.")
        }
        return w
    }

    private func primaryHeight() -> CGFloat { NSScreen.screens.first?.frame.height ?? 0 }

    /// The NSScreen holding a point of the top-left global space.
    private func screenContaining(_ point: CGPoint) -> NSScreen? {
        let h = primaryHeight()
        return NSScreen.screens.first { s in
            let f = s.frame
            return CGRect(x: f.minX, y: h - f.maxY, width: f.width, height: f.height).contains(point)
        } ?? NSScreen.main
    }

    func screens() -> [[String: Any]] {
        let h = primaryHeight()
        return NSScreen.screens.enumerated().map { index, s in
            let f = s.frame
            let id = (s.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value ?? 0
            return ["id": String(id), "x": Int(f.minX), "y": Int(h - f.maxY), "width": Int(f.width), "height": Int(f.height),
                    "scale": Double(s.backingScaleFactor), "primary": index == 0]
        }
    }

    func available() -> Bool {
        guard let session = CGSessionCopyCurrentDictionary() as? [String: Any] else { return false }
        if (session[kCGSessionOnConsoleKey as String] as? Bool) == false { return false }
        return (session["CGSSessionScreenIsLocked"] as? Bool) != true
    }

    // MARK: accessibility

    private func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
        var value: CFTypeRef?
        return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil
    }
    private func element(_ element: AXUIElement, _ name: String) -> AXUIElement? {
        guard let value = attribute(element, name), CFGetTypeID(value) == AXUIElementGetTypeID() else { return nil }
        return (value as! AXUIElement)
    }
    private func frame(_ element: AXUIElement) -> CGRect? {
        guard let p = attribute(element, kAXPositionAttribute), CFGetTypeID(p) == AXValueGetTypeID(),
              let s = attribute(element, kAXSizeAttribute), CFGetTypeID(s) == AXValueGetTypeID() else { return nil }
        var point = CGPoint.zero, size = CGSize.zero
        guard AXValueGetValue(p as! AXValue, .cgPoint, &point), AXValueGetValue(s as! AXValue, .cgSize, &size) else { return nil }
        return CGRect(origin: point, size: size)
    }
    private func isSecure(_ element: AXUIElement) -> Bool {
        (attribute(element, kAXSubroleAttribute) as? String) == "AXSecureTextField" || (attribute(element, "AXProtectedContent") as? Bool) == true
    }

    /// The AX window of a CGWindow: the one AX window of that app with the same frame (never a guess).
    private func axWindow(_ w: WorkerWindow) -> AXUIElement? {
        let root = AXUIElementCreateApplication(w.pid)
        AXUIElementSetMessagingTimeout(root, 0.25)
        if (attribute(root, "AXManualAccessibility") as? Bool) == false {
            // Electron apps expose their tree on request (an assistive-technology handshake, not a permission).
            AXUIElementSetAttributeValue(root, "AXManualAccessibility" as CFString, kCFBooleanTrue)
        }
        let windows = attribute(root, kAXWindowsAttribute) as? [AXUIElement] ?? []
        let matches = windows.filter { element in
            guard let f = frame(element) else { return false }
            return abs(f.minX - w.bounds.minX) < 2 && abs(f.minY - w.bounds.minY) < 2 && abs(f.width - w.bounds.width) < 2 && abs(f.height - w.bounds.height) < 2
        }
        guard matches.count == 1, let match = matches.first else { return nil }
        AXUIElementSetMessagingTimeout(match, 0.25)
        return match
    }

    /// Frames of password and protected fields in the window (values never read).
    private func secureFrames(_ root: AXUIElement) -> [CGRect] {
        var out: [CGRect] = []
        var seen = 0
        let started = ProcessInfo.processInfo.systemUptime
        func walk(_ element: AXUIElement, _ depth: Int) {
            guard depth < 28, seen < 2500, ProcessInfo.processInfo.systemUptime - started < 0.6 else { return }
            seen += 1
            if isSecure(element) { if let f = frame(element) { out.append(f) }; return }
            for child in attribute(element, kAXChildrenAttribute) as? [AXUIElement] ?? [] { walk(child, depth + 1) }
        }
        walk(root, 0)
        return out
    }

    /// The focused element of the app, when it is inside the approved window.
    private func focusedElement(_ w: WorkerWindow, _ ax: AXUIElement?) -> AXUIElement? {
        let root = AXUIElementCreateApplication(w.pid)
        guard let focused = element(root, kAXFocusedUIElementAttribute) else { return nil }
        if let ax, let owner = element(focused, kAXWindowAttribute), !CFEqual(owner, ax) { return nil }
        return focused
    }

    private func caret(_ w: WorkerWindow, _ ax: AXUIElement?) -> CGRect? {
        guard let focused = focusedElement(w, ax), let range = attribute(focused, kAXSelectedTextRangeAttribute) else { return nil }
        var bounds: CFTypeRef?
        guard AXUIElementCopyParameterizedAttributeValue(focused, kAXBoundsForRangeParameterizedAttribute as CFString, range, &bounds) == .success,
              let bounds, CFGetTypeID(bounds) == AXValueGetTypeID() else { return nil }
        var rect = CGRect.zero
        guard AXValueGetValue(bounds as! AXValue, .cgRect, &rect) else { return nil }
        return CGRect(x: rect.minX, y: rect.minY, width: max(2, rect.width), height: max(2, rect.height))
    }

    // MARK: capture

    private func scWindow(_ id: CGWindowID) async throws -> SCWindow {
        let now = ProcessInfo.processInfo.systemUptime
        if let cache = scCache, now - cache.at < 2, let window = cache.windows[id] { return window }
        let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: true)
        var windows: [CGWindowID: SCWindow] = [:]
        for window in content.windows { windows[window.windowID] = window }
        scCache = (now, windows)
        guard let window = windows[id] else { throw UseError("capture_unavailable", "The window image is unavailable. Keep it visible and observe again.") }
        return window
    }

    func capture(_ w: WorkerWindow, maxLongSide: Int) async throws -> ([String: Any], Data) {
        let shareable = try await scWindow(w.id)
        guard shareable.owningApplication?.processID == w.pid else { throw UseError("window_changed", "The window changed. Observe again.") }
        let filter = SCContentFilter(desktopIndependentWindow: shareable)
        let size = WorkerGeometry.captureSize(points: w.bounds.size, scale: CGFloat(filter.pointPixelScale), maxLongSide: maxLongSide)
        let config = SCStreamConfiguration()
        config.width = size.width; config.height = size.height
        config.showsCursor = false
        config.ignoreShadowsSingleWindow = true
        let image: CGImage
        do { image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) }
        catch { scCache = nil; throw UseError("capture_unavailable", "The window image is unavailable. Keep it visible and observe again.") }
        let width = image.width, height = image.height
        var pixels = Data(count: width * height * 4)
        let drawn = pixels.withUnsafeMutableBytes { buffer -> Bool in
            guard let context = CGContext(data: buffer.baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4,
                                          space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return false }
            // Opaque: transparent window corners become white instead of black.
            context.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1))
            context.fill(CGRect(x: 0, y: 0, width: width, height: height))
            context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
            return true
        }
        guard drawn else { throw UseError("capture_unavailable", "The window image could not be read.") }
        // The window must not have moved while it was captured.
        let after = try window(w.id)
        guard after.identity == w.identity, after.bounds == w.bounds else { throw UseError("window_changed", "The approved window moved or changed. Observe it again.") }
        let ax = axWindow(w)
        let secure = (ax.map(secureFrames) ?? []).compactMap { WorkerGeometry.pixels(of: $0, bounds: w.bounds, width: width, height: height, pad: 2) }
        let caretRect = caret(w, ax).flatMap { WorkerGeometry.pixels(of: $0, bounds: w.bounds, width: width, height: height, pad: 4) }
        let lines = await recognize(image, pixels: pixels)
        var header: [String: Any] = ["width": width, "height": height, "secure": secure.map(WorkerGeometry.json),
            "text_boxes": lines ?? [], "ocr": lines == nil ? "none" : "client", "source": "window", "secure_checked": ax != nil]
        if let caretRect { header["caret"] = WorkerGeometry.json(caretRect) } else { header["caret"] = NSNull() }
        return (header, pixels)
    }

    /// On-device text recognition (Vision, fast): lines and word boxes in pixels; reused while the pixels are unchanged.
    private func recognize(_ image: CGImage, pixels: Data) async -> [[String: Any]]? {
        let digest = Data(SHA256.hash(data: pixels))
        if let cache = ocrCache, cache.digest == digest { return cache.lines }
        let width = image.width, height = image.height
        let encoded: Data = await Task.detached(priority: .userInitiated) {
            let request = VNRecognizeTextRequest()
            request.recognitionLevel = .fast
            request.usesLanguageCorrection = false
            do { try VNImageRequestHandler(cgImage: image, options: [:]).perform([request]) } catch { return Data("null".utf8) }
            let lines: [[String: Any]] = (request.results ?? []).compactMap { observation -> [String: Any]? in
                guard let candidate = observation.topCandidates(1).first else { return nil }
                let text = candidate.string
                let words: [[String: Any]] = WorkerGeometry.words(text).compactMap { word, range in
                    guard let box = try? candidate.boundingBox(for: range)?.boundingBox else { return nil }
                    return ["text": word, "box": WorkerGeometry.json(WorkerGeometry.pixels(ofNormalized: box, width: width, height: height))]
                }
                return ["text": text, "box": WorkerGeometry.json(WorkerGeometry.pixels(ofNormalized: observation.boundingBox, width: width, height: height)), "words": words]
            }
            return (try? JSONSerialization.data(withJSONObject: lines)) ?? Data("[]".utf8)
        }.value
        let lines = (try? JSONSerialization.jsonObject(with: encoded, options: [.fragmentsAllowed])) as? [[String: Any]]
        ocrCache = (digest, lines)
        return lines
    }

    // MARK: hit test, focus, checks

    /// True when the topmost visible window at `point` (screen points) is the approved window.
    func hit(_ w: WorkerWindow, _ point: CGPoint) -> Bool {
        guard w.bounds.contains(point) else { return false }
        let infos = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
        for info in infos {
            guard ((info[kCGWindowAlpha as String] as? NSNumber)?.doubleValue ?? 1) > 0.01,
                  let raw = info[kCGWindowBounds as String] as? NSDictionary,
                  let bounds = CGRect(dictionaryRepresentation: raw as CFDictionary), bounds.contains(point) else { continue }
            return (info[kCGWindowNumber as String] as? NSNumber)?.uint32Value == w.id
        }
        return false
    }

    func focus(_ w: WorkerWindow) async throws {
        guard let app = NSRunningApplication(processIdentifier: w.pid) else { throw UseError("window_unavailable", "The approved app has closed.") }
        if let ax = axWindow(w) { AXUIElementPerformAction(ax, kAXRaiseAction as CFString) }
        app.activate(options: [])
        for _ in 0..<20 {
            if frontmost(w) { return }
            try await Task.sleep(nanoseconds: 50_000_000)
        }
        throw UseError("focus_changed", "Bring the approved app to the front, then choose Continue.")
    }

    private func frontmost(_ w: WorkerWindow) -> Bool {
        guard NSWorkspace.shared.frontmostApplication?.processIdentifier == w.pid else { return false }
        guard let ax = axWindow(w) else { return true }
        guard let focused = element(AXUIElementCreateApplication(w.pid), kAXFocusedWindowAttribute) else { return true }
        return CFEqual(focused, ax)
    }

    @discardableResult
    func check(_ w: WorkerWindow, generation expected: Int, deadline: Double) throws -> WorkerWindow {
        if deadline > 0 && Date().timeIntervalSince1970 * 1000 >= deadline { throw UseError("session_expired", "The approved session expired. Start a new session.") }
        if generation != expected { throw UseError("paused_by_person", "Your input paused computer use. Choose Continue when ready.") }
        let current = try window(w.id)
        guard current.identity == w.identity, current.bounds == w.bounds else { throw UseError("window_changed", "The approved window moved or changed. Observe it again.") }
        guard frontmost(current) else { throw UseError("focus_changed", "The approved window must stay in front. Choose Continue when ready.") }
        return current
    }

    // MARK: input

    private func mouse(_ type: NSEvent.EventType, _ point: CGPoint, _ w: WorkerWindow, clicks: Int, flags: CGEventFlags) throws {
        guard let setWindowLocation = MacInput.setWindowLocation else {
            throw UseError("unsupported_action", "This macOS version does not support window-scoped pointer input.")
        }
        let frame = w.bounds
        let pressed = type == .leftMouseDown || type == .rightMouseDown || type == .leftMouseDragged
        guard let native = NSEvent.mouseEvent(with: type, location: NSPoint(x: point.x - frame.minX, y: frame.maxY - point.y),
                modifierFlags: NSEvent.ModifierFlags(rawValue: UInt(flags.rawValue)), timestamp: ProcessInfo.processInfo.systemUptime,
                windowNumber: Int(w.id), context: nil, eventNumber: 0, clickCount: clicks, pressure: pressed ? 1 : 0),
              let event = native.cgEvent?.copy() else { throw UseError("input_interrupted", "Could not create a window-targeted mouse event.") }
        event.location = point
        event.flags = flags
        setWindowLocation(event, CGPoint(x: point.x - frame.minX, y: point.y - frame.minY))
        event.setIntegerValueField(.mouseEventButtonNumber, value: type == .rightMouseDown || type == .rightMouseUp ? 1 : 0)
        event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(w.id))
        event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(w.id))
        event.setIntegerValueField(.mouseEventClickState, value: Int64(clicks))
        post(event, w.pid)
    }

    /// Tracks a pressed button during a drag (reported, so the host can release it if this process dies).
    private func hold(_ point: CGPoint, _ w: WorkerWindow, _ flags: CGEventFlags) {
        let first = held == nil
        held = HeldPointer(pid: w.pid, point: point, window: w.id, frame: w.bounds, flags: flags)
        if first { reportHeld() }
    }

    private func post(_ event: CGEvent, _ pid: pid_t) {
        event.setIntegerValueField(.eventSourceUnixProcessID, value: Int64(ownPid))
        event.postToPid(pid)
    }

    private func key(_ code: CGKeyCode, flags: CGEventFlags, text: String?, pid: pid_t) throws {
        guard let down = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
              let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false) else {
            throw UseError("input_interrupted", "Could not create a keyboard event.")
        }
        down.flags = flags; up.flags = flags
        if let text {
            let units = Array(text.utf16)
            units.withUnsafeBufferPointer { buffer in
                guard let base = buffer.baseAddress else { return }
                down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: base)
                up.keyboardSetUnicodeString(stringLength: units.count, unicodeString: base)
            }
        }
        // Down and up are posted together: a key is never left half sent.
        post(down, pid); post(up, pid)
    }

    private func reportHeld() {
        var message: [String: Any] = ["event": "held", "keys": [Int](), "buttons": held == nil ? [Int]() : [1]]
        if let held {
            message["release"] = ["pid": Int(held.pid), "window": Int(held.window), "x": Double(held.point.x), "y": Double(held.point.y),
                                  "frame": [Double(held.frame.minX), Double(held.frame.minY), Double(held.frame.width), Double(held.frame.height)]]
        }
        send(message)
    }

    private func releaseHeld() {
        guard let h = held else { return }
        held = nil
        let w = WorkerWindow(id: h.window, pid: h.pid, identity: "", executable: "", appId: "", appName: "", title: "", bounds: h.frame, scale: 1, display: 0)
        try? mouse(.leftMouseUp, h.point, w, clicks: 1, flags: h.flags)
        reportHeld()
    }

    private static func point(_ value: Any?) throws -> CGPoint {
        guard let p = value as? [String: Any], let x = number(p["x"]), let y = number(p["y"]) else { throw UseError("invalid_action", "A point needs x and y.") }
        return CGPoint(x: x, y: y)
    }

    func act(_ w: WorkerWindow, generation expected: Int, action: [String: Any], deadline: Double) async throws {
        let checked = { () throws -> Void in _ = try self.check(w, generation: expected, deadline: deadline) }
        let type = action["type"] as? String ?? ""
        let modifiers = action["modifiers"] as? [String] ?? []
        guard let flags = WorkerKeys.flags(modifiers) else { throw UseError("invalid_action", "Modifiers are ctrl, shift, alt and meta.") }
        defer { releaseHeld() }
        try checked()
        switch type {
        case "move":
            let p = try Self.point(action)
            try mouse(.mouseMoved, p, w, clicks: 0, flags: flags)
        case "click", "double_click", "right_click":
            let p = try Self.point(action)
            let (down, up): (NSEvent.EventType, NSEvent.EventType) = type == "right_click" ? (.rightMouseDown, .rightMouseUp) : (.leftMouseDown, .leftMouseUp)
            let clicks = type == "double_click" ? 2 : 1
            for click in 1...clicks {
                if click > 1 { try await Task.sleep(nanoseconds: 70_000_000); try checked() }
                try mouse(down, p, w, clicks: click, flags: flags)
                try mouse(up, p, w, clicks: click, flags: flags)
            }
        case "scroll":
            let p = try Self.point(action)
            guard let delta = Self.int(action["delta"]), delta != 0, abs(delta) <= 20 else { throw UseError("invalid_action", "Scroll needs delta: 1 to 20 steps.") }
            // Positive delta = down/right; one step ≈ three lines.
            let amount = Int32(-delta * 48)
            let horizontal = action["axis"] as? String == "horizontal"
            guard let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: horizontal ? 0 : amount, wheel2: horizontal ? amount : 0, wheel3: 0) else {
                throw UseError("input_interrupted", "Could not create a scroll event.")
            }
            event.location = p
            event.flags = flags
            event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(w.id))
            event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(w.id))
            post(event, w.pid)
        case "drag":
            let raw: [Any] = action["path"] as? [Any] ?? []
            let path: [CGPoint] = try raw.map { try Self.point($0) }
            guard path.count >= 2, path.count <= 64 else { throw UseError("invalid_action", "A drag needs 2 to 64 points.") }
            hold(path[0], w, flags)
            try mouse(.leftMouseDown, path[0], w, clicks: 1, flags: flags)
            for p in path.dropFirst() {
                try await Task.sleep(nanoseconds: 16_000_000)
                try checked()
                hold(p, w, flags)
                try mouse(.leftMouseDragged, p, w, clicks: 1, flags: flags)
            }
            try await Task.sleep(nanoseconds: 16_000_000)
            try mouse(.leftMouseUp, path[path.count - 1], w, clicks: 1, flags: flags)
            held = nil; reportHeld()
        case "type":
            guard let text = action["text"] as? String, !text.isEmpty else { throw UseError("invalid_action", "Text is required.") }
            let ax = axWindow(w)
            for character in text {
                try checked()
                if let focused = focusedElement(w, ax), isSecure(focused) {
                    throw UseError("protected_input", "This is a password field. Enter passwords yourself.")
                }
                try key(0, flags: [], text: String(character), pid: w.pid)
                try await Task.sleep(nanoseconds: 4_000_000)
            }
        case "key":
            guard let name = action["key"] as? String, let resolved = WorkerKeys.resolve(name) else { throw UseError("unsupported_action", "This key is not available on macOS.") }
            if WorkerKeys.blocked(name, modifiers) { throw UseError("unsupported_action", "System-wide shortcuts are unavailable.") }
            var keyFlags = flags
            if resolved.shift { keyFlags.insert(.maskShift) }
            // With ctrl/meta held, apps read the key code; plain keys also carry their text.
            let text = keyFlags.contains(.maskCommand) || keyFlags.contains(.maskControl) ? nil : resolved.text
            try key(resolved.code, flags: keyFlags, text: text, pid: w.pid)
        default:
            throw UseError("unsupported_action", "This action is unavailable.")
        }
    }

    // MARK: person input

    private func startMonitoring(interrupt: Bool) {
        interrupting = interrupt || interrupting
        if interrupt { anchor = CGEvent(source: nil)?.location }
        guard monitors.isEmpty else { return }
        let presses: NSEvent.EventTypeMask = [.leftMouseDown, .rightMouseDown, .otherMouseDown, .keyDown, .scrollWheel, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged]
        if let monitor = NSEvent.addGlobalMonitorForEvents(matching: presses.union(.mouseMoved), handler: { [weak self] event in
            MainActor.assumeIsolated { self?.observe(event) }
        }) { monitors.append(monitor) }
    }

    private func stopMonitoring() {
        for monitor in monitors { NSEvent.removeMonitor(monitor) }
        monitors.removeAll(); interrupting = false; anchor = nil
    }

    private func observe(_ event: NSEvent) {
        let cg = event.cgEvent
        let sourcePid = cg?.getIntegerValueField(.eventSourceUnixProcessID) ?? 0
        if sourcePid == Int64(ownPid) { return } // our own window-targeted input
        if event.type == .mouseMoved {
            // Passive motion counts only when the pointer clearly leaves where it was when control started.
            guard interrupting, let anchor, let now = cg?.location, hypot(now.x - anchor.x, now.y - anchor.y) > 8 else { return }
            personInput(force: false); return
        }
        if [.leftMouseDown, .rightMouseDown, .otherMouseDown, .keyDown].contains(event.type) {
            let stateID = cg?.getIntegerValueField(.eventSourceStateID) ?? -1
            let synthetic = WorkerInputSource.isSynthetic(sourcePid: sourcePid, stateID: stateID, sourceIsWindowServer: Self.isWindowServer(pid_t(truncatingIfNeeded: sourcePid)))
            var entry: [String: Any] = ["synthetic": synthetic, "at": Int(Date().timeIntervalSince1970 * 1000), "kind": event.type == .keyDown ? "key" : "pointer"]
            // The app in front at the press, where the pointer was (global points, top-left origin) and the key as a terminal names it.
            entry["foreground_pid"] = NSWorkspace.shared.frontmostApplication.map { Int($0.processIdentifier) } ?? NSNull()
            if let at = cg?.location, event.type != .keyDown { entry["x"] = Double(at.x); entry["y"] = Double(at.y) }
            entry["key"] = event.type == .keyDown ? (Self.keyName(event) ?? NSNull()) : NSNull()
            lastInput = entry
        }
        personInput(force: false)
    }

    /** One lower-case letter or digit for printable keys, else Enter / Escape / Space / Tab / Backspace / arrows; nil otherwise. */
    static func keyName(_ event: NSEvent) -> String? {
        switch event.keyCode {
        case 36, 76: return "Enter"
        case 53: return "Escape"
        case 49: return "Space"
        case 48: return "Tab"
        case 51: return "Backspace"
        case 123: return "Left"
        case 124: return "Right"
        case 125: return "Down"
        case 126: return "Up"
        default:
            guard let chars = event.charactersIgnoringModifiers?.lowercased(), chars.count == 1, let c = chars.first, c.isLetter || c.isNumber else { return nil }
            return String(c)
        }
    }

    private static func isWindowServer(_ pid: pid_t) -> Bool {
        guard pid > 0 else { return false }
        var name = [CChar](repeating: 0, count: 256)
        guard proc_name(pid, &name, UInt32(name.count)) > 0 else { return false }
        return String(cString: name) == "WindowServer"
    }

    private func personInput(force: Bool) {
        guard interrupting || force else { return }
        generation += 1
        releaseHeld()
        anchor = CGEvent(source: nil)?.location
        send(["event": "person-input", "generation": generation])
    }

    // MARK: release after a crash

    /// `ComputerUse release '<held json>'`: lifts a mouse button a crashed worker left pressed.
    static func releaseOnly(_ json: String) -> Bool {
        guard let data = json.data(using: .utf8), let value = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
              let pid = int(value["pid"]), pid > 0, let window = int(value["window"]), let x = number(value["x"]), let y = number(value["y"]),
              let f = value["frame"] as? [NSNumber], f.count == 4, let setWindowLocation = MacInput.setWindowLocation else { return false }
        let frame = CGRect(x: f[0].doubleValue, y: f[1].doubleValue, width: f[2].doubleValue, height: f[3].doubleValue)
        let point = CGPoint(x: x, y: y)
        guard let native = NSEvent.mouseEvent(with: .leftMouseUp, location: NSPoint(x: point.x - frame.minX, y: frame.maxY - point.y), modifierFlags: [],
                timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: window, context: nil, eventNumber: 0, clickCount: 1, pressure: 0),
              let event = native.cgEvent?.copy() else { return false }
        event.location = point
        setWindowLocation(event, CGPoint(x: point.x - frame.minX, y: point.y - frame.minY))
        event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(window))
        event.setIntegerValueField(.eventSourceUnixProcessID, value: Int64(ProcessInfo.processInfo.processIdentifier))
        event.postToPid(pid_t(pid))
        return true
    }
}