Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
7.2 KB
Raw Blame
import Foundation
import CoreGraphics

struct UseError: LocalizedError {
    let code: String
    let message: String
    let next: String
    init(_ code: String, _ message: String, next: String = "correct_request") {
        self.code = code; self.message = message; self.next = next
    }
    var errorDescription: String? { message }
    var payload: [String: Any] { ["ok": false, "code": code, "message": message, "next": next] }
}

enum AccessMode: String, CaseIterable {
    case observe, assist, control
    var title: String {
        switch self {
        case .observe: return "Read this window"
        case .assist: return "Use app controls"
        case .control: return "Use mouse and keyboard"
        }
    }
    var explanation: String {
        switch self {
        case .observe: return "Read window text and screenshots. No clicks or typing."
        case .assist: return "Read this window and use its accessible controls. Your pointer stays free."
        case .control: return "Read this window and use its controls, mouse and keyboard while it is in front. Local input pauses the session."
        }
    }
}

struct Arguments {
    let values: [String: Any]
    func only(_ keys: Set<String>) throws {
        guard Set(values.keys).isSubset(of: keys) else { throw UseError("invalid_arguments", "Unexpected argument. Read the tool schema.") }
    }
    func string(_ key: String, max: Int = 256) throws -> String {
        guard let value = values[key] as? String, !value.isEmpty, value.utf8.count <= max,
              !value.unicodeScalars.contains(where: { $0.value == 0 }) else {
            throw UseError("invalid_arguments", "\(key) must be a non-empty string of at most \(max) bytes.")
        }
        return value
    }
    func number(_ key: String, min: Double, max: Double) throws -> Double {
        guard let value = values[key] as? NSNumber, CFGetTypeID(value) != CFBooleanGetTypeID(),
              value.doubleValue.isFinite, value.doubleValue >= min, value.doubleValue <= max else {
            throw UseError("invalid_arguments", "\(key) must be a finite number between \(min) and \(max).")
        }
        return value.doubleValue
    }
    func integer(_ key: String, min: Int, max: Int) throws -> Int {
        let value = try number(key, min: Double(min), max: Double(max))
        guard value.rounded() == value else { throw UseError("invalid_arguments", "\(key) must be an integer.") }
        return Int(value)
    }
    func bool(_ key: String, default fallback: Bool) throws -> Bool {
        guard let value = values[key] else { return fallback }
        guard let number = value as? NSNumber, CFGetTypeID(number) == CFBooleanGetTypeID() else {
            throw UseError("invalid_arguments", "\(key) must be a boolean.")
        }
        return number.boolValue
    }
}

enum Action {
    case press(String), setValue(String, String), key(String), type(String)
    case click(CGPoint, Int), scroll(CGPoint, Int32, Int32), drag([CGPoint])

    init(_ values: [String: Any]) throws {
        let a = Arguments(values: values)
        let type = try a.string("type")
        switch type {
        case "press":
            try a.only(["type", "ref"]); self = .press(try a.string("ref"))
        case "set_value":
            try a.only(["type", "ref", "text"])
            // Empty text is a meaningful replacement.
            guard let text = values["text"] as? String, text.utf8.count <= 8_000, !text.contains("\0") else {
                throw UseError("invalid_arguments", "text must be a string of at most 8000 bytes.")
            }
            self = .setValue(try a.string("ref"), text)
        case "key":
            try a.only(["type", "key"])
            let key = try a.string("key")
            guard NativeKey.allowed.contains(key) else { throw UseError("unsupported_key", "Use one of the keys listed by computer_discover. System shortcuts are unavailable.") }
            self = .key(key)
        case "type":
            try a.only(["type", "text"]); self = .type(try a.string("text", max: 8_000))
        case "click", "double_click":
            try a.only(["type", "x", "y"]); self = .click(try Self.point(values), type == "click" ? 1 : 2)
        case "scroll":
            try a.only(["type", "x", "y", "delta_x", "delta_y"])
            self = .scroll(try Self.point(values), Int32(try a.integer("delta_x", min: -1200, max: 1200)), Int32(try a.integer("delta_y", min: -1200, max: 1200)))
        case "drag":
            try a.only(["type", "path"])
            guard let path = values["path"] as? [[String: Any]], (2...32).contains(path.count) else {
                throw UseError("invalid_arguments", "path needs 2–32 image points.")
            }
            self = .drag(try path.map { point in try Arguments(values: point).only(["x", "y"]); return try Self.point(point) })
        default: throw UseError("unknown_action", "Unsupported action. Read computer_act's schema.")
        }
    }
    private static func point(_ values: [String: Any]) throws -> CGPoint {
        let a = Arguments(values: values)
        return CGPoint(x: try a.number("x", min: 0, max: 100_000), y: try a.number("y", min: 0, max: 100_000))
    }
    var name: String {
        switch self {
        case .press: return "press"
        case .setValue: return "set_value"
        case .key: return "key"
        case .type: return "type"
        case .click(_, let count): return count == 1 ? "click" : "double_click"
        case .scroll: return "scroll"
        case .drag: return "drag"
        }
    }
    var requiresPointer: Bool {
        switch self { case .press, .setValue: return false; default: return true }
    }
}

enum NativeKey {
    // No global app switcher, launcher, pasteboard, permissions, or shell shortcuts.
    static let codes: [String: CGKeyCode] = ["enter": 36, "tab": 48, "escape": 53, "backspace": 51,
        "delete": 117, "left": 123, "right": 124, "down": 125, "up": 126,
        "home": 115, "end": 119, "page_up": 116, "page_down": 121, "space": 49]
    static let allowed = Set(codes.keys).union(["select_all", "undo", "redo"])
}

struct ObservationLease {
    let id: String
    let createdAt: TimeInterval
    let generation: Int
    let frame: CGRect
    let imageWidth: Int
    let imageHeight: Int
    var stateDigest: Data = Data()
    var imageDigest: Data?

    func validate(id requested: String, generation current: Int, frame currentFrame: CGRect, now: TimeInterval) throws {
        guard requested == id, generation == current, now - createdAt <= 15,
              now >= createdAt, frame == currentFrame else {
            throw UseError("stale_observation", "The window or observation changed. Observe again before acting.", next: "observe")
        }
    }
    func screenPoint(_ point: CGPoint) throws -> CGPoint {
        guard point.x.isFinite, point.y.isFinite, imageWidth > 0, imageHeight > 0,
              point.x >= 0, point.y >= 0, point.x < CGFloat(imageWidth), point.y < CGFloat(imageHeight) else {
            throw UseError("outside_window", "Coordinates must be inside the current observation image.", next: "observe")
        }
        return CGPoint(x: frame.minX + point.x * frame.width / CGFloat(imageWidth),
                       y: frame.minY + point.y * frame.height / CGFloat(imageHeight))
    }
}