Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(lume): add Claude computer-use agent mode for Setup Assistant automation (#1217)
Francesco Bonacci2172bd9986aa
diff --git a/libs/lume/src/Commands/Setup.swift b/libs/lume/src/Commands/Setup.swiftindex da9c285d2c..d40823cfec 100644--- a/libs/lume/src/Commands/Setup.swift+++ b/libs/lume/src/Commands/Setup.swift@@ -5,8 +5,13 @@ struct Setup: AsyncParsableCommand {static let configuration = CommandConfiguration(abstract: "[Preview] Run unattended Setup Assistant automation on a macOS VM",discussion: """- This is an experimental feature. Unattended configurations are specific to macOS \- versions and may not work across different releases.+ This is an experimental feature. Two modes are available:++ - preset: Uses a YAML config file with scripted click/type commands (default).+ Presets are specific to macOS versions and may break across releases.++ - agent: Uses Claude's computer-use API to intelligently navigate the Setup Assistant.+ Adapts to any macOS version. Requires an Anthropic API key.The VM must be freshly created and not have completed the Setup Assistant yet."""@@ -17,11 +22,36 @@ struct Setup: AsyncParsableCommand {completion: .custom(completeVMName))var name: String+ @Option(+ name: .customLong("mode"),+ help: "Setup mode: 'preset' for YAML-based automation, 'agent' for Claude computer-use agent (default: preset)")+ var mode: String = "preset"+@Option(name: .customLong("unattended"),- help: "Preset name or path to YAML config file for unattended macOS Setup Assistant automation. Built-in presets: sequoia, tahoe.",+ help: "Preset name or path to YAML config file (required for preset mode). Built-in presets: sequoia, tahoe.",completion: .file(extensions: ["yml", "yaml"]))- var unattended: String+ var unattended: String?++ @Option(+ name: .customLong("anthropic-key"),+ help: "Anthropic API key for agent mode. Can also be set via ANTHROPIC_API_KEY env var.")+ var anthropicKey: String?++ @Option(+ name: .customLong("model"),+ help: "Claude model to use for agent mode (default: claude-sonnet-4-6)")+ var model: String = "claude-sonnet-4-6"++ @Option(+ name: .customLong("max-iterations"),+ help: "Maximum agent iterations before giving up (default: 200)")+ var maxIterations: Int = 200++ @Option(+ name: .customLong("system-prompt"),+ help: "Custom system prompt for the agent (overrides the default)")+ var systemPrompt: String?@Option(name: .customLong("storage"),@@ -55,33 +85,71 @@ struct Setup: AsyncParsableCommand {func run() async throws {Logger.info("[Preview] Unattended setup is an experimental feature")- // Load unattended config- let config: UnattendedConfig- let isPreset = UnattendedConfig.isPreset(name: unattended)- do {- config = try UnattendedConfig.load(from: unattended)- Logger.info("Loaded unattended config", metadata: [- "source": isPreset ? "preset:\(unattended)" : unattended,- "bootWait": "\(config.bootWait)s",- "commands": "\(config.bootCommands.count)"+ switch mode {+ case "agent":+ // Resolve API key from flag or environment+ let apiKey = anthropicKey ?? ProcessInfo.processInfo.environment["ANTHROPIC_API_KEY"]+ guard let apiKey, !apiKey.isEmpty else {+ throw UnattendedError.configLoadFailed(+ "Agent mode requires an Anthropic API key. Provide --anthropic-key or set ANTHROPIC_API_KEY env var."+ )+ }++ TelemetryClient.shared.record(event: TelemetryEvent.setup, properties: [+ "mode": "agent",+ "model": model])- } catch {- throw UnattendedError.configLoadFailed("Failed to load config from \(unattended): \(error.localizedDescription)")- }- // Record telemetry- TelemetryClient.shared.record(event: TelemetryEvent.setup, properties: [- "preset_name": isPreset ? unattended : "custom"- ])-- try await LumeController().setup(- name: name,- config: config,- storage: storage,- vncPort: vncPort,- noDisplay: noDisplay,- debug: debug,- debugDir: debugDir- )+ try await LumeController().setupWithAgent(+ name: name,+ apiKey: apiKey,+ model: model,+ maxIterations: maxIterations,+ systemPrompt: systemPrompt,+ storage: storage,+ vncPort: vncPort,+ noDisplay: noDisplay,+ debug: debug,+ debugDir: debugDir+ )++ case "preset":+ guard let unattended else {+ throw UnattendedError.configLoadFailed(+ "Preset mode requires --unattended <preset|path>. Built-in presets: sequoia, tahoe."+ )+ }++ let config: UnattendedConfig+ let isPreset = UnattendedConfig.isPreset(name: unattended)+ do {+ config = try UnattendedConfig.load(from: unattended)+ Logger.info("Loaded unattended config", metadata: [+ "source": isPreset ? "preset:\(unattended)" : unattended,+ "bootWait": "\(config.bootWait)s",+ "commands": "\(config.bootCommands.count)"+ ])+ } catch {+ throw UnattendedError.configLoadFailed("Failed to load config from \(unattended): \(error.localizedDescription)")+ }++ TelemetryClient.shared.record(event: TelemetryEvent.setup, properties: [+ "mode": "preset",+ "preset_name": isPreset ? unattended : "custom"+ ])++ try await LumeController().setup(+ name: name,+ config: config,+ storage: storage,+ vncPort: vncPort,+ noDisplay: noDisplay,+ debug: debug,+ debugDir: debugDir+ )++ default:+ throw UnattendedError.configLoadFailed("Unknown setup mode '\(mode)'. Use 'preset' or 'agent'.")+ }}}diff --git a/libs/lume/src/LumeController.swift b/libs/lume/src/LumeController.swiftindex 0aadcc28ae..2a6ce0fe99 100644--- a/libs/lume/src/LumeController.swift+++ b/libs/lume/src/LumeController.swift@@ -802,6 +802,57 @@ final class LumeController {}}+ /// Run agent-based Setup Assistant automation using Claude computer-use API+ @MainActor+ public func setupWithAgent(+ name: String,+ apiKey: String,+ model: String = "claude-sonnet-4-6",+ maxIterations: Int = 100,+ systemPrompt: String? = nil,+ storage: String? = nil,+ vncPort: Int = 0,+ noDisplay: Bool = false,+ debug: Bool = false,+ debugDir: String? = nil+ ) async throws {+ let normalizedName = normalizeVMName(name: name)+ Logger.info(+ "Running agent-based setup",+ metadata: [+ "name": normalizedName,+ "model": model,+ "maxIterations": "\(maxIterations)",+ "debug": "\(debug)"+ ])++ do {+ let vm = try get(name: normalizedName, storage: storage)++ guard vm.config.os.lowercased() == "macos" else {+ throw VMError.unsupportedOS("Unattended setup is only supported for macOS VMs, got: \(vm.config.os)")+ }++ let installer = UnattendedInstaller()+ try await installer.installWithAgent(+ vm: vm,+ apiKey: apiKey,+ model: model,+ maxIterations: maxIterations,+ systemPrompt: systemPrompt,+ vncPort: vncPort,+ noDisplay: noDisplay,+ debug: debug,+ debugDir: debugDir+ )++ Logger.info("Agent-based setup completed", metadata: ["name": normalizedName])+ } catch {+ Logger.error("Failed to run agent-based setup", metadata: ["error": error.localizedDescription])+ throw error+ }+ }+@MainActorpublic func delete(name: String, storage: String? = nil) async throws {let normalizedName = normalizeVMName(name: name)diff --git a/libs/lume/src/Unattended/AgentSetupRunner.swift b/libs/lume/src/Unattended/AgentSetupRunner.swiftnew file mode 100644index 0000000000..e9acbac354--- /dev/null+++ b/libs/lume/src/Unattended/AgentSetupRunner.swift@@ -0,0 +1,566 @@+import Foundation+import CoreGraphics+import ImageIO++/// Runs macOS Setup Assistant automation using Claude's computer-use API+/// instead of brittle YAML presets. Adapts to any macOS version.+@MainActor+final class AgentSetupRunner {+ private let vncService: VNCService+ private let client: AnthropicClient+ /// Display dimensions reported to Claude (capped for API efficiency)+ private let displayWidth: Int+ private let displayHeight: Int+ /// Actual VNC framebuffer dimensions (for coordinate mapping)+ private let vncWidth: Int+ private let vncHeight: Int+ private let maxIterations: Int+ private let debug: Bool+ private let debugDirectory: URL+ private var stepIndex: Int = 0+ /// Optional SSH reachability check — returns true if SSH is reachable+ private let sshCheck: (() async -> Bool)?++ /// Max dimensions to report to Claude API (keeps screenshot payloads small)+ private static let maxAPIWidth = 1024+ private static let maxAPIHeight = 768++ static let defaultSystemPrompt = """+ You are automating the macOS Setup Assistant on a freshly installed virtual machine.+ Your goal is to complete the setup as quickly as possible with these settings:++ 1. Language: English+ 2. Country: United States+ 3. Skip all Apple ID sign-in (look for "Set Up Later", "Skip", "Other Sign-In Options", or similar)+ 4. Create a local user account with:+ - Full Name: lume+ - Account Name: lume+ - Password: lume+ 5. Skip or decline all optional features (Siri, Analytics, Screen Time, Location Services, etc.)+ 6. Accept any privacy/terms screens by clicking Continue/Agree+ 7. Reach the macOS desktop and enable SSH++ ## Expected screen sequence (adapt as needed — screens may vary by macOS version):+ 1. "Hello" greeting animation → press Space or click to dismiss+ 2. Language selection → type "English", select it, press Enter+ 3. Country/Region → type "United States", press Enter, click Continue+ 4. Transfer Your Data → click "Set up as new", then Continue+ 5. Written and Spoken Languages → click Continue+ 6. Accessibility → click "Not Now"+ 7. Data & Privacy → click Continue+ 8. Create a Mac Account → fill Full Name "lume", Tab to skip Account Name (auto-fills), type password "lume", Tab, verify password "lume", Tab to skip Hint, Tab to checkbox, press Space to untoggle "Allow reset with Apple Account", click Continue+ 9. Apple Account sign-in → look for "Set Up Later" or "Other Sign-In Options" or "Skip". If a "Don't Skip" confirmation appears, Tab to "Skip" and press Enter+ 10. Terms and Conditions → click "Agree" button (at bottom, not in the text), then click "Agree" again in the confirmation popup+ 11. Enable Location Services → click Continue, then confirm "Don't Use" in popup+ 12. Select Your Time Zone → click Continue+ 13. Analytics → click Continue+ 14. Screen Time → click "Set Up Later"+ 15. Siri → click to enable/disable, then click Continue+ 16. FileVault → click "Not Now", then confirm in popup+ 17. Choose Your Look → click Continue+ 18. Update Mac Automatically → click Continue+ 19. Welcome/Get Started → click "Get Started"++ ## After reaching the desktop, enable SSH:+ 1. Open Spotlight (Cmd+Space), type "System Settings", press Enter+ 2. Search for "Remote Login" using Cmd+F, click "Remote Login"+ 3. Enable the Remote Login toggle, allow all users, click Done+ 4. Close System Settings (Cmd+Q)+ 5. The task is now complete.++ ## Important tips:+ - The screenshot colors may appear tinted/shifted — this is a VNC artifact. Focus on shapes and layout.+ - After each action, take a screenshot to verify the result before proceeding.+ - If a button or text is not visible, try waiting 2-3 seconds, then take another screenshot.+ - Use keyboard shortcuts when buttons are hard to click (Tab to navigate, Enter to confirm, Space to toggle).+ - If you see a password confirmation field, enter "lume" again.+ - When you see the macOS desktop with the Dock and menu bar, proceed to enable SSH.+ - When you see the lock screen with the user "lume", the setup is complete — just stop.+ """++ init(+ vncService: VNCService,+ apiKey: String,+ model: String = "claude-sonnet-4-6",+ displayWidth: Int,+ displayHeight: Int,+ maxIterations: Int = 200,+ debug: Bool = false,+ debugDirectory: URL? = nil,+ sshCheck: (() async -> Bool)? = nil+ ) {+ self.vncService = vncService+ self.client = AnthropicClient(apiKey: apiKey, model: model)+ self.vncWidth = displayWidth+ self.vncHeight = displayHeight+ // Cap display dimensions for API efficiency+ self.displayWidth = min(displayWidth, Self.maxAPIWidth)+ self.displayHeight = min(displayHeight, Self.maxAPIHeight)+ self.maxIterations = maxIterations+ self.debug = debug+ self.debugDirectory = debugDirectory ?? FileManager.default.temporaryDirectory+ .appendingPathComponent("agent-setup-\(UUID().uuidString)")+ self.sshCheck = sshCheck+ }++ /// Run the agent loop to complete Setup Assistant+ func run(systemPrompt: String? = nil) async throws {+ let prompt = systemPrompt ?? Self.defaultSystemPrompt++ if debug {+ try? FileManager.default.createDirectory(at: debugDirectory, withIntermediateDirectories: true)+ Logger.info("Agent debug screenshots will be saved to \(debugDirectory.path)")+ }++ // Take initial screenshot+ Logger.info("Agent taking initial screenshot")+ let initialScreenshot = try await captureScreenshot()++ // Build initial messages with screenshot+ var messages: [[String: Any]] = [+ [+ "role": "user",+ "content": [+ [+ "type": "text",+ "text": "Here is the current screen. Please complete the macOS Setup Assistant."+ ] as [String: Any],+ [+ "type": "image",+ "source": [+ "type": "base64",+ "media_type": "image/jpeg",+ "data": initialScreenshot+ ]+ ] as [String: Any]+ ]+ ]+ ]++ var iteration = 0+ while iteration < maxIterations {+ iteration += 1+ Logger.info("Agent iteration \(iteration)/\(maxIterations)")++ // Call Claude API+ let response = try await client.sendMessage(+ systemPrompt: prompt,+ messages: messages,+ displayWidth: displayWidth,+ displayHeight: displayHeight+ )++ // Add assistant response to conversation+ let assistantContent = try responseToRawContent(response)+ messages.append(["role": "assistant", "content": assistantContent])++ // Check if Claude is done (no tool use)+ let toolUseBlocks = response.content.filter {+ if case .toolUse = $0 { return true }+ return false+ }++ if toolUseBlocks.isEmpty {+ // Claude thinks the task is complete+ for block in response.content {+ if case .text(let text) = block {+ Logger.info("Agent completed: \(text)")+ }+ }+ Logger.info("Agent setup finished after \(iteration) iterations")+ return+ }++ // Process each tool use+ var toolResults: [[String: Any]] = []+ for block in response.content {+ guard case .toolUse(let id, let name, let input) = block else { continue }+ guard name == "computer" else {+ toolResults.append([+ "type": "tool_result",+ "tool_use_id": id,+ "content": "Unknown tool: \(name)",+ "is_error": true+ ])+ continue+ }++ let result = try await executeComputerAction(id: id, input: input)+ toolResults.append(result)+ }++ // Add tool results to messages+ messages.append(["role": "user", "content": toolResults])++ // Check SSH reachability — if SSH is up, setup is complete regardless of agent state.+ // Only start checking after step 50 (setup assistant takes at least that many) and+ // check every 10 iterations to avoid slowing down the loop.+ if iteration >= 50 && iteration % 10 == 0, let sshCheck = sshCheck {+ if await sshCheck() {+ Logger.info("SSH is reachable — setup complete after \(iteration) iterations")+ return+ }+ }++ // Keep conversation history bounded to prevent request_too_large errors.+ // Keep the first message (initial screenshot) plus the last 20 messages (10 turns).+ let maxHistory = 20+ if messages.count > maxHistory + 1 {+ let first = messages[0]+ let recent = Array(messages.suffix(maxHistory))+ messages = [first] + recent+ }+ }++ throw AnthropicError.maxIterationsReached+ }++ // MARK: - Action Execution++ private func executeComputerAction(id: String, input: [String: AnyCodable]) async throws -> [String: Any] {+ guard let action = input["action"]?.stringValue else {+ return errorResult(id: id, message: "Missing 'action' field")+ }++ Logger.info("Agent action: \(action)", metadata: inputMetadata(input))+ stepIndex += 1++ do {+ switch action {+ case "screenshot":+ let base64 = try await captureScreenshot()+ return screenshotResult(id: id, base64: base64)++ case "left_click":+ guard let coord = coordinateFromInput(input) else {+ return errorResult(id: id, message: "Missing coordinate for left_click")+ }+ try await vncService.sendMouseClick(at: coord, button: .left)+ try await Task.sleep(nanoseconds: 500_000_000) // 500ms for UI to update+ let base64 = try await captureScreenshot()+ return screenshotResult(id: id, base64: base64)++ case "right_click":+ guard let coord = coordinateFromInput(input) else {+ return errorResult(id: id, message: "Missing coordinate for right_click")+ }+ try await vncService.sendMouseClick(at: coord, button: .right)+ try await Task.sleep(nanoseconds: 500_000_000)+ let base64 = try await captureScreenshot()+ return screenshotResult(id: id, base64: base64)++ case "double_click":+ guard let coord = coordinateFromInput(input) else {+ return errorResult(id: id, message: "Missing coordinate for double_click")+ }+ try await vncService.sendMouseClick(at: coord, button: .left)+ try await Task.sleep(nanoseconds: 100_000_000)+ try await vncService.sendMouseClick(at: coord, button: .left)+ try await Task.sleep(nanoseconds: 500_000_000)+ let base64 = try await captureScreenshot()+ return screenshotResult(id: id, base64: base64)++ case "type":+ guard let text = input["text"]?.stringValue else {+ return errorResult(id: id, message: "Missing text for type action")+ }+ try await vncService.sendText(text, delayMs: 50)+ try await Task.sleep(nanoseconds: 300_000_000)+ let base64 = try await captureScreenshot()+ return screenshotResult(id: id, base64: base64)++ case "key":+ guard let keyText = input["text"]?.stringValue else {+ return errorResult(id: id, message: "Missing text for key action")+ }+ try await sendKey(keyText)+ try await Task.sleep(nanoseconds: 300_000_000)+ let base64 = try await captureScreenshot()+ return screenshotResult(id: id, base64: base64)++ case "scroll":+ // For scroll, we just take a screenshot after — VNC scroll support varies+ guard let coord = coordinateFromInput(input) else {+ return errorResult(id: id, message: "Missing coordinate for scroll")+ }+ let direction = input["scroll_direction"]?.stringValue ?? "down"+ let amount = input["scroll_amount"]?.intValue ?? 3+ try await sendScroll(at: coord, direction: direction, amount: amount)+ try await Task.sleep(nanoseconds: 500_000_000)+ let base64 = try await captureScreenshot()+ return screenshotResult(id: id, base64: base64)++ case "mouse_move":+ // Just acknowledge — VNC doesn't need explicit moves before clicks+ return [+ "type": "tool_result",+ "tool_use_id": id,+ "content": "Mouse moved"+ ]++ case "wait":+ let duration = input["duration"]?.intValue ?? 2+ try await Task.sleep(nanoseconds: UInt64(duration) * 1_000_000_000)+ let base64 = try await captureScreenshot()+ return screenshotResult(id: id, base64: base64)++ default:+ return errorResult(id: id, message: "Unsupported action: \(action)")+ }+ } catch {+ Logger.error("Agent action failed", metadata: ["action": action, "error": error.localizedDescription])+ return errorResult(id: id, message: "Action '\(action)' failed: \(error.localizedDescription)")+ }+ }++ // MARK: - Screenshot++ private func captureScreenshot() async throws -> String {+ let image = try await vncService.captureFramebuffer()++ // Normalize and resize: re-render through a standard RGBA CGContext.+ // This fixes VNC color channel issues (BGRA→RGBA) and scales to display size.+ let normalized = normalizeImage(image, maxWidth: displayWidth, maxHeight: displayHeight)++ if debug {+ saveDebugScreenshot(normalized)+ }++ // Encode as JPEG for much smaller payloads (PNG screenshots can be 3MB+)+ guard let base64 = normalized.jpegBase64(quality: 0.7) else {+ throw UnattendedError.framebufferCaptureFailed("Failed to encode screenshot as JPEG")+ }++ return base64+ }++ /// Resize the image to fit within the given bounds. Let CoreGraphics handle pixel format+ /// conversion naturally — no manual channel swapping needed.+ private func normalizeImage(_ image: CGImage, maxWidth: Int, maxHeight: Int) -> CGImage {+ let srcWidth = image.width+ let srcHeight = image.height++ // Scale to fit within bounds+ let scale: Double+ if srcWidth > maxWidth || srcHeight > maxHeight {+ scale = min(Double(maxWidth) / Double(srcWidth), Double(maxHeight) / Double(srcHeight))+ } else {+ scale = 1.0+ }+ let newWidth = Int(Double(srcWidth) * scale)+ let newHeight = Int(Double(srcHeight) * scale)++ let colorSpace = CGColorSpaceCreateDeviceRGB()+ guard let context = CGContext(+ data: nil,+ width: newWidth,+ height: newHeight,+ bitsPerComponent: 8,+ bytesPerRow: 0,+ space: colorSpace,+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue+ ) else {+ return image+ }++ context.interpolationQuality = .high+ context.draw(image, in: CGRect(x: 0, y: 0, width: newWidth, height: newHeight))++ return context.makeImage() ?? image+ }++ private func saveDebugScreenshot(_ image: CGImage) {+ let filename = String(format: "%03d-agent-step.png", stepIndex)+ let url = debugDirectory.appendingPathComponent(filename)+ guard let destination = CGImageDestinationCreateWithURL(url as CFURL, "public.png" as CFString, 1, nil) else {+ return+ }+ CGImageDestinationAddImage(destination, image, nil)+ CGImageDestinationFinalize(destination)+ Logger.info("Saved debug screenshot", metadata: ["path": url.path])+ }++ // MARK: - Input Helpers++ /// Parse coordinate from Claude's response and scale to VNC framebuffer space.+ /// Claude operates in displayWidth x displayHeight space, VNC uses vncWidth x vncHeight.+ private func coordinateFromInput(_ input: [String: AnyCodable]) -> CGPoint? {+ if let coordArray = input["coordinate"]?.value as? [Any], coordArray.count == 2 {+ let x: Double+ let y: Double+ if let xi = coordArray[0] as? Int, let yi = coordArray[1] as? Int {+ x = Double(xi)+ y = Double(yi)+ } else if let xd = coordArray[0] as? Double, let yd = coordArray[1] as? Double {+ x = xd+ y = yd+ } else {+ return nil+ }+ // Scale from Claude's coordinate space to VNC framebuffer space+ let scaleX = Double(vncWidth) / Double(displayWidth)+ let scaleY = Double(vncHeight) / Double(displayHeight)+ return CGPoint(x: x * scaleX, y: y * scaleY)+ }+ return nil+ }++ private func sendKey(_ keyText: String) async throws {+ // Parse key combinations like "ctrl+s", "Return", "space", etc.+ let parts = keyText.lowercased().split(separator: "+").map(String.init)++ var modifiers: VNCKeyModifiers = .none+ var mainKey: String = ""++ for part in parts {+ switch part {+ case "ctrl", "control": modifiers.insert(.control)+ case "shift": modifiers.insert(.shift)+ case "alt", "option": modifiers.insert(.option)+ case "cmd", "command", "super", "meta": modifiers.insert(.command)+ default: mainKey = part+ }+ }++ let keyCode = keyNameToKeyCode(mainKey)+ try await vncService.sendKeyPress(keyCode, modifiers: modifiers)+ }++ private func sendScroll(at point: CGPoint, direction: String, amount: Int) async throws {+ // VNC scroll is button 4 (up) or button 5 (down)+ // Use key-based scrolling as fallback (VNC scroll button support varies)+ for _ in 0..<amount {+ switch direction {+ case "up":+ try await vncService.sendKeyPress(keyNameToKeyCode("up"), modifiers: .none)+ case "down":+ try await vncService.sendKeyPress(keyNameToKeyCode("down"), modifiers: .none)+ case "left":+ try await vncService.sendKeyPress(keyNameToKeyCode("left"), modifiers: .none)+ case "right":+ try await vncService.sendKeyPress(keyNameToKeyCode("right"), modifiers: .none)+ default:+ break+ }+ try await Task.sleep(nanoseconds: 100_000_000)+ }+ }++ private func keyNameToKeyCode(_ name: String) -> UInt16 {+ // Map common key names to macOS key codes+ switch name {+ case "return", "enter": return 36+ case "tab": return 48+ case "space": return 49+ case "escape", "esc": return 53+ case "backspace", "delete": return 51+ case "up": return 126+ case "down": return 125+ case "left": return 123+ case "right": return 124+ case "a": return 0+ case "b": return 11+ case "c": return 8+ case "d": return 2+ case "e": return 14+ case "f": return 3+ case "g": return 5+ case "h": return 4+ case "i": return 34+ case "j": return 38+ case "k": return 40+ case "l": return 37+ case "m": return 46+ case "n": return 45+ case "o": return 31+ case "p": return 35+ case "q": return 12+ case "r": return 15+ case "s": return 1+ case "t": return 17+ case "u": return 32+ case "v": return 9+ case "w": return 13+ case "x": return 7+ case "y": return 16+ case "z": return 6+ case "f1": return 122+ case "f2": return 120+ case "f3": return 99+ case "f4": return 118+ case "f5": return 96+ case "f6": return 97+ case "f7": return 98+ case "f8": return 100+ case "f9": return 101+ case "f10": return 109+ case "f11": return 103+ case "f12": return 111+ default: return 49 // space as fallback+ }+ }++ // MARK: - Result Builders++ private func screenshotResult(id: String, base64: String) -> [String: Any] {+ return [+ "type": "tool_result",+ "tool_use_id": id,+ "content": [+ [+ "type": "image",+ "source": [+ "type": "base64",+ "media_type": "image/jpeg",+ "data": base64+ ]+ ] as [String: Any]+ ]+ ]+ }++ private func errorResult(id: String, message: String) -> [String: Any] {+ return [+ "type": "tool_result",+ "tool_use_id": id,+ "content": message,+ "is_error": true+ ]+ }++ private func inputMetadata(_ input: [String: AnyCodable]) -> [String: String] {+ var meta: [String: String] = [:]+ if let action = input["action"]?.stringValue { meta["action"] = action }+ if let text = input["text"]?.stringValue { meta["text"] = text }+ if let coord = input["coordinate"]?.value as? [Any] {+ meta["coordinate"] = "\(coord)"+ }+ return meta+ }++ // MARK: - Response Serialization++ private func responseToRawContent(_ response: AnthropicClient.APIResponse) throws -> [[String: Any]] {+ var content: [[String: Any]] = []+ for block in response.content {+ switch block {+ case .text(let text):+ content.append(["type": "text", "text": text])+ case .toolUse(let id, let name, let input):+ var inputDict: [String: Any] = [:]+ for (key, val) in input {+ inputDict[key] = val.value+ }+ content.append([+ "type": "tool_use",+ "id": id,+ "name": name,+ "input": inputDict+ ])+ default:+ break+ }+ }+ return content+ }+}diff --git a/libs/lume/src/Unattended/AnthropicClient.swift b/libs/lume/src/Unattended/AnthropicClient.swiftnew file mode 100644index 0000000000..7296b110a7--- /dev/null+++ b/libs/lume/src/Unattended/AnthropicClient.swift@@ -0,0 +1,240 @@+import Foundation+import CoreGraphics+import ImageIO++/// Minimal Anthropic API client for computer-use agent loop+struct AnthropicClient {+ let apiKey: String+ let model: String+ let betaFlag: String = "computer-use-2025-11-24"++ private let baseURL = URL(string: "https://api.anthropic.com/v1/messages")!++ struct Message: Codable {+ let role: String+ let content: [ContentBlock]+ }++ enum ContentBlock: Codable {+ case text(String)+ case image(base64: String, mediaType: String)+ case toolUse(id: String, name: String, input: [String: AnyCodable])+ case toolResult(toolUseId: String, content: [ContentBlock], isError: Bool)++ enum CodingKeys: String, CodingKey {+ case type, text, source, id, name, input+ case toolUseId = "tool_use_id"+ case content, isError = "is_error"+ }++ func encode(to encoder: Encoder) throws {+ var container = encoder.container(keyedBy: CodingKeys.self)+ switch self {+ case .text(let text):+ try container.encode("text", forKey: .type)+ try container.encode(text, forKey: .text)+ case .image(let base64, let mediaType):+ try container.encode("image", forKey: .type)+ let source: [String: String] = [+ "type": "base64",+ "media_type": mediaType,+ "data": base64+ ]+ try container.encode(source, forKey: .source)+ case .toolUse(let id, let name, let input):+ try container.encode("tool_use", forKey: .type)+ try container.encode(id, forKey: .id)+ try container.encode(name, forKey: .name)+ try container.encode(input, forKey: .input)+ case .toolResult(let toolUseId, let content, let isError):+ try container.encode("tool_result", forKey: .type)+ try container.encode(toolUseId, forKey: .toolUseId)+ try container.encode(content, forKey: .content)+ if isError {+ try container.encode(true, forKey: .isError)+ }+ }+ }++ init(from decoder: Decoder) throws {+ let container = try decoder.container(keyedBy: CodingKeys.self)+ let type = try container.decode(String.self, forKey: .type)+ switch type {+ case "text":+ let text = try container.decode(String.self, forKey: .text)+ self = .text(text)+ case "tool_use":+ let id = try container.decode(String.self, forKey: .id)+ let name = try container.decode(String.self, forKey: .name)+ let input = try container.decode([String: AnyCodable].self, forKey: .input)+ self = .toolUse(id: id, name: name, input: input)+ default:+ self = .text("[unsupported block type: \(type)]")+ }+ }+ }++ struct APIResponse: Decodable {+ let id: String+ let content: [ContentBlock]+ let stopReason: String?++ enum CodingKeys: String, CodingKey {+ case id, content+ case stopReason = "stop_reason"+ }+ }++ struct APIError: Decodable {+ let type: String+ let error: ErrorDetail++ struct ErrorDetail: Decodable {+ let type: String+ let message: String+ }+ }++ @MainActor+ func sendMessage(+ systemPrompt: String,+ messages: [[String: Any]],+ displayWidth: Int,+ displayHeight: Int+ ) async throws -> APIResponse {+ var request = URLRequest(url: baseURL)+ request.httpMethod = "POST"+ request.setValue("application/json", forHTTPHeaderField: "content-type")+ request.setValue(apiKey, forHTTPHeaderField: "x-api-key")+ request.setValue("2023-06-01", forHTTPHeaderField: "anthropic-version")+ request.setValue(betaFlag, forHTTPHeaderField: "anthropic-beta")+ request.timeoutInterval = 120++ let body: [String: Any] = [+ "model": model,+ "max_tokens": 4096,+ "system": systemPrompt,+ "tools": [+ [+ "type": "computer_20251124",+ "name": "computer",+ "display_width_px": displayWidth,+ "display_height_px": displayHeight+ ] as [String: Any]+ ],+ "messages": messages+ ]++ request.httpBody = try JSONSerialization.data(withJSONObject: body)++ let (data, response) = try await URLSession.shared.data(for: request)++ guard let httpResponse = response as? HTTPURLResponse else {+ throw AnthropicError.invalidResponse+ }++ if httpResponse.statusCode != 200 {+ if let errorResponse = try? JSONDecoder().decode(APIError.self, from: data) {+ throw AnthropicError.apiError(+ status: httpResponse.statusCode,+ message: errorResponse.error.message+ )+ }+ let body = String(data: data, encoding: .utf8) ?? "unknown"+ throw AnthropicError.apiError(status: httpResponse.statusCode, message: body)+ }++ return try JSONDecoder().decode(APIResponse.self, from: data)+ }+}++enum AnthropicError: Error, LocalizedError {+ case invalidResponse+ case apiError(status: Int, message: String)+ case maxIterationsReached++ var errorDescription: String? {+ switch self {+ case .invalidResponse:+ return "Invalid response from Anthropic API"+ case .apiError(let status, let message):+ return "Anthropic API error (\(status)): \(message)"+ case .maxIterationsReached:+ return "Agent reached maximum iteration limit"+ }+ }+}++/// Type-erased Codable wrapper for JSON values+struct AnyCodable: Codable {+ let value: Any++ init(_ value: Any) {+ self.value = value+ }++ init(from decoder: Decoder) throws {+ let container = try decoder.singleValueContainer()+ if let intVal = try? container.decode(Int.self) {+ value = intVal+ } else if let doubleVal = try? container.decode(Double.self) {+ value = doubleVal+ } else if let stringVal = try? container.decode(String.self) {+ value = stringVal+ } else if let boolVal = try? container.decode(Bool.self) {+ value = boolVal+ } else if let arrayVal = try? container.decode([AnyCodable].self) {+ value = arrayVal.map { $0.value }+ } else if let dictVal = try? container.decode([String: AnyCodable].self) {+ value = dictVal.mapValues { $0.value }+ } else {+ value = NSNull()+ }+ }++ func encode(to encoder: Encoder) throws {+ var container = encoder.singleValueContainer()+ switch value {+ case let val as Int: try container.encode(val)+ case let val as Double: try container.encode(val)+ case let val as String: try container.encode(val)+ case let val as Bool: try container.encode(val)+ case is NSNull: try container.encodeNil()+ default: try container.encodeNil()+ }+ }++ var stringValue: String? { value as? String }+ var intValue: Int? { value as? Int }+ var doubleValue: Double? { value as? Double }+ var arrayValue: [Any]? { value as? [Any] }+}++// MARK: - CGImage to base64++extension CGImage {+ func pngBase64() -> String? {+ let mutableData = NSMutableData()+ guard let destination = CGImageDestinationCreateWithData(mutableData, "public.png" as CFString, 1, nil) else {+ return nil+ }+ CGImageDestinationAddImage(destination, self, nil)+ guard CGImageDestinationFinalize(destination) else {+ return nil+ }+ return (mutableData as Data).base64EncodedString()+ }++ func jpegBase64(quality: Double = 0.8) -> String? {+ let mutableData = NSMutableData()+ guard let destination = CGImageDestinationCreateWithData(mutableData, "public.jpeg" as CFString, 1, nil) else {+ return nil+ }+ let options: [CFString: Any] = [kCGImageDestinationLossyCompressionQuality: quality]+ CGImageDestinationAddImage(destination, self, options as CFDictionary)+ guard CGImageDestinationFinalize(destination) else {+ return nil+ }+ return (mutableData as Data).base64EncodedString()+ }+}diff --git a/libs/lume/src/Unattended/UnattendedInstaller.swift b/libs/lume/src/Unattended/UnattendedInstaller.swiftindex 70f5eebc3a..95e842570e 100644--- a/libs/lume/src/Unattended/UnattendedInstaller.swift+++ b/libs/lume/src/Unattended/UnattendedInstaller.swift@@ -5,14 +5,136 @@ import Virtualization@MainActorfinal class UnattendedInstaller {- /// Run unattended setup on a VM- /// - Parameters:- /// - vm: The VM to configure- /// - config: Unattended configuration with boot commands- /// - vncPort: VNC port to use (0 for auto-assign)- /// - noDisplay: If true, don't open the VNC client- /// - debug: If true, save screenshots with click coordinates- /// - debugDir: Custom directory for debug screenshots (defaults to unique folder in system temp)+ /// Run agent-based setup using Claude computer-use API+ func installWithAgent(+ vm: VM,+ apiKey: String,+ model: String = "claude-sonnet-4-6",+ maxIterations: Int = 100,+ systemPrompt: String? = nil,+ vncPort: Int = 0,+ noDisplay: Bool = false,+ debug: Bool = false,+ debugDir: String? = nil+ ) async throws {+ Logger.info("Starting agent-based setup", metadata: [+ "vm": vm.name,+ "model": model,+ "maxIterations": "\(maxIterations)",+ "debug": "\(debug)"+ ])++ // Start the VM (same retry logic as preset mode)+ Logger.info("Starting VM for agent setup (background task)")+ var vmStartError: Error?+ let vmTask = Task {+ var attempts = 0+ let maxAttempts = 3+ while attempts < maxAttempts {+ do {+ try await vm.run(+ noDisplay: noDisplay,+ sharedDirectories: [],+ mount: nil,+ vncPort: vncPort,+ recoveryMode: false,+ usbMassStoragePaths: []+ )+ return+ } catch {+ let errorDescription = error.localizedDescription+ if errorDescription.contains("auxiliary storage") || errorDescription.contains("Failed to lock") {+ attempts += 1+ if attempts < maxAttempts {+ Logger.info("VM start failed due to auxiliary storage lock, retrying", metadata: [+ "attempt": "\(attempts)/\(maxAttempts)",+ "waitTime": "5s"+ ])+ try await Task.sleep(nanoseconds: 5_000_000_000)+ continue+ }+ }+ vmStartError = error+ throw error+ }+ }+ }++ // Give the VM a moment to start+ try await Task.sleep(nanoseconds: 2_000_000_000)++ if let error = vmStartError {+ Logger.error("VM failed to start", metadata: ["error": error.localizedDescription])+ throw error+ }++ // Wait for boot (agent mode uses a fixed 60s boot wait)+ let bootWait: UInt64 = 60+ Logger.info("Waiting for VM to boot", metadata: ["bootWait": "\(bootWait)s"])+ try await Task.sleep(nanoseconds: bootWait * 1_000_000_000)++ // Connect VNC input client+ Logger.info("Connecting VNC input client for agent automation")+ try await vm.vncService.connectInputClient()++ // Get display dimensions from VNC framebuffer+ let framebufferSize = await vm.vncService.getFrameBufferSize() ?? (width: 1920, height: 1440)+ let displayWidth = Int(framebufferSize.width)+ let displayHeight = Int(framebufferSize.height)++ Logger.info("VNC framebuffer size", metadata: [+ "width": "\(displayWidth)",+ "height": "\(displayHeight)"+ ])++ // Create SSH reachability check closure+ let macAddress = vm.vmDirContext.config.macAddress+ let sshCheck: (() async -> Bool)? = macAddress != nil ? {+ guard let ip = DHCPLeaseParser.getIPAddress(forMAC: macAddress!) else {+ return false+ }+ Logger.info("Checking SSH reachability", metadata: ["ip": ip])+ let runner = HealthCheckRunner()+ let check = HealthCheck(type: "ssh", user: "lume", password: "lume", timeout: 5, retries: 1, retryDelay: 0)+ return (try? await runner.run(check: check, vmIP: ip)) ?? false+ } : nil++ // Create and run the agent+ let debugDirectory: URL? = debugDir.map { URL(fileURLWithPath: $0) }+ let agent = AgentSetupRunner(+ vncService: vm.vncService,+ apiKey: apiKey,+ model: model,+ displayWidth: displayWidth,+ displayHeight: displayHeight,+ maxIterations: maxIterations,+ debug: debug,+ debugDirectory: debugDirectory,+ sshCheck: sshCheck+ )++ do {+ try await agent.run(systemPrompt: systemPrompt)+ Logger.info("Agent setup completed successfully")++ vm.vncService.disconnectInputClient()++ // Stop the VM+ Logger.info("Agent setup finished - stopping VM")+ vmTask.cancel()+ try? await vm.stop()+ Logger.info("VM stopped", metadata: ["name": vm.name])+ } catch {+ Logger.error("Agent setup failed", metadata: ["error": error.localizedDescription])+ vm.vncService.disconnectInputClient()+ vmTask.cancel()+ try? await vm.stop()+ Logger.info("VM stopped after failure", metadata: ["name": vm.name])+ throw error+ }+ }++ /// Run unattended setup on a VM using preset commandsfunc install(vm: VM,config: UnattendedConfig,