Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(lume): add Claude computer-use agent mode for Setup Assistant automation (#1217)

Francesco Bonacci2172bd9986aa
diff --git a/libs/lume/src/Commands/Setup.swift b/libs/lume/src/Commands/Setup.swift
index da9c285d2c..d40823cfec 100644
--- a/libs/lume/src/Commands/Setup.swift
+++ b/libs/lume/src/Commands/Setup.swift
@@ -5,8 +5,13 @@ struct Setup: AsyncParsableCommand {
static let configuration = CommandConfiguration(
abstract: "[Preview] Run unattended Setup Assistant automation on a macOS VM",
discussion: """
- This is an experimental feature. Unattended configurations are specific to macOS \
- versions and may not work across different releases.
+ This is an experimental feature. Two modes are available:
+
+ - preset: Uses a YAML config file with scripted click/type commands (default).
+ Presets are specific to macOS versions and may break across releases.
+
+ - agent: Uses Claude's computer-use API to intelligently navigate the Setup Assistant.
+ Adapts to any macOS version. Requires an Anthropic API key.
The VM must be freshly created and not have completed the Setup Assistant yet.
"""
@@ -17,11 +22,36 @@ struct Setup: AsyncParsableCommand {
completion: .custom(completeVMName))
var name: String
+ @Option(
+ name: .customLong("mode"),
+ help: "Setup mode: 'preset' for YAML-based automation, 'agent' for Claude computer-use agent (default: preset)")
+ var mode: String = "preset"
+
@Option(
name: .customLong("unattended"),
- help: "Preset name or path to YAML config file for unattended macOS Setup Assistant automation. Built-in presets: sequoia, tahoe.",
+ help: "Preset name or path to YAML config file (required for preset mode). Built-in presets: sequoia, tahoe.",
completion: .file(extensions: ["yml", "yaml"]))
- var unattended: String
+ var unattended: String?
+
+ @Option(
+ name: .customLong("anthropic-key"),
+ help: "Anthropic API key for agent mode. Can also be set via ANTHROPIC_API_KEY env var.")
+ var anthropicKey: String?
+
+ @Option(
+ name: .customLong("model"),
+ help: "Claude model to use for agent mode (default: claude-sonnet-4-6)")
+ var model: String = "claude-sonnet-4-6"
+
+ @Option(
+ name: .customLong("max-iterations"),
+ help: "Maximum agent iterations before giving up (default: 200)")
+ var maxIterations: Int = 200
+
+ @Option(
+ name: .customLong("system-prompt"),
+ help: "Custom system prompt for the agent (overrides the default)")
+ var systemPrompt: String?
@Option(
name: .customLong("storage"),
@@ -55,33 +85,71 @@ struct Setup: AsyncParsableCommand {
func run() async throws {
Logger.info("[Preview] Unattended setup is an experimental feature")
- // Load unattended config
- let config: UnattendedConfig
- let isPreset = UnattendedConfig.isPreset(name: unattended)
- do {
- config = try UnattendedConfig.load(from: unattended)
- Logger.info("Loaded unattended config", metadata: [
- "source": isPreset ? "preset:\(unattended)" : unattended,
- "bootWait": "\(config.bootWait)s",
- "commands": "\(config.bootCommands.count)"
+ switch mode {
+ case "agent":
+ // Resolve API key from flag or environment
+ let apiKey = anthropicKey ?? ProcessInfo.processInfo.environment["ANTHROPIC_API_KEY"]
+ guard let apiKey, !apiKey.isEmpty else {
+ throw UnattendedError.configLoadFailed(
+ "Agent mode requires an Anthropic API key. Provide --anthropic-key or set ANTHROPIC_API_KEY env var."
+ )
+ }
+
+ TelemetryClient.shared.record(event: TelemetryEvent.setup, properties: [
+ "mode": "agent",
+ "model": model
])
- } catch {
- throw UnattendedError.configLoadFailed("Failed to load config from \(unattended): \(error.localizedDescription)")
- }
- // Record telemetry
- TelemetryClient.shared.record(event: TelemetryEvent.setup, properties: [
- "preset_name": isPreset ? unattended : "custom"
- ])
-
- try await LumeController().setup(
- name: name,
- config: config,
- storage: storage,
- vncPort: vncPort,
- noDisplay: noDisplay,
- debug: debug,
- debugDir: debugDir
- )
+ try await LumeController().setupWithAgent(
+ name: name,
+ apiKey: apiKey,
+ model: model,
+ maxIterations: maxIterations,
+ systemPrompt: systemPrompt,
+ storage: storage,
+ vncPort: vncPort,
+ noDisplay: noDisplay,
+ debug: debug,
+ debugDir: debugDir
+ )
+
+ case "preset":
+ guard let unattended else {
+ throw UnattendedError.configLoadFailed(
+ "Preset mode requires --unattended <preset|path>. Built-in presets: sequoia, tahoe."
+ )
+ }
+
+ let config: UnattendedConfig
+ let isPreset = UnattendedConfig.isPreset(name: unattended)
+ do {
+ config = try UnattendedConfig.load(from: unattended)
+ Logger.info("Loaded unattended config", metadata: [
+ "source": isPreset ? "preset:\(unattended)" : unattended,
+ "bootWait": "\(config.bootWait)s",
+ "commands": "\(config.bootCommands.count)"
+ ])
+ } catch {
+ throw UnattendedError.configLoadFailed("Failed to load config from \(unattended): \(error.localizedDescription)")
+ }
+
+ TelemetryClient.shared.record(event: TelemetryEvent.setup, properties: [
+ "mode": "preset",
+ "preset_name": isPreset ? unattended : "custom"
+ ])
+
+ try await LumeController().setup(
+ name: name,
+ config: config,
+ storage: storage,
+ vncPort: vncPort,
+ noDisplay: noDisplay,
+ debug: debug,
+ debugDir: debugDir
+ )
+
+ default:
+ throw UnattendedError.configLoadFailed("Unknown setup mode '\(mode)'. Use 'preset' or 'agent'.")
+ }
}
}
diff --git a/libs/lume/src/LumeController.swift b/libs/lume/src/LumeController.swift
index 0aadcc28ae..2a6ce0fe99 100644
--- a/libs/lume/src/LumeController.swift
+++ b/libs/lume/src/LumeController.swift
@@ -802,6 +802,57 @@ final class LumeController {
}
}
+ /// Run agent-based Setup Assistant automation using Claude computer-use API
+ @MainActor
+ public func setupWithAgent(
+ name: String,
+ apiKey: String,
+ model: String = "claude-sonnet-4-6",
+ maxIterations: Int = 100,
+ systemPrompt: String? = nil,
+ storage: String? = nil,
+ vncPort: Int = 0,
+ noDisplay: Bool = false,
+ debug: Bool = false,
+ debugDir: String? = nil
+ ) async throws {
+ let normalizedName = normalizeVMName(name: name)
+ Logger.info(
+ "Running agent-based setup",
+ metadata: [
+ "name": normalizedName,
+ "model": model,
+ "maxIterations": "\(maxIterations)",
+ "debug": "\(debug)"
+ ])
+
+ do {
+ let vm = try get(name: normalizedName, storage: storage)
+
+ guard vm.config.os.lowercased() == "macos" else {
+ throw VMError.unsupportedOS("Unattended setup is only supported for macOS VMs, got: \(vm.config.os)")
+ }
+
+ let installer = UnattendedInstaller()
+ try await installer.installWithAgent(
+ vm: vm,
+ apiKey: apiKey,
+ model: model,
+ maxIterations: maxIterations,
+ systemPrompt: systemPrompt,
+ vncPort: vncPort,
+ noDisplay: noDisplay,
+ debug: debug,
+ debugDir: debugDir
+ )
+
+ Logger.info("Agent-based setup completed", metadata: ["name": normalizedName])
+ } catch {
+ Logger.error("Failed to run agent-based setup", metadata: ["error": error.localizedDescription])
+ throw error
+ }
+ }
+
@MainActor
public func delete(name: String, storage: String? = nil) async throws {
let normalizedName = normalizeVMName(name: name)
diff --git a/libs/lume/src/Unattended/AgentSetupRunner.swift b/libs/lume/src/Unattended/AgentSetupRunner.swift
new file mode 100644
index 0000000000..e9acbac354
--- /dev/null
+++ b/libs/lume/src/Unattended/AgentSetupRunner.swift
@@ -0,0 +1,566 @@
+import Foundation
+import CoreGraphics
+import ImageIO
+
+/// Runs macOS Setup Assistant automation using Claude's computer-use API
+/// instead of brittle YAML presets. Adapts to any macOS version.
+@MainActor
+final class AgentSetupRunner {
+ private let vncService: VNCService
+ private let client: AnthropicClient
+ /// Display dimensions reported to Claude (capped for API efficiency)
+ private let displayWidth: Int
+ private let displayHeight: Int
+ /// Actual VNC framebuffer dimensions (for coordinate mapping)
+ private let vncWidth: Int
+ private let vncHeight: Int
+ private let maxIterations: Int
+ private let debug: Bool
+ private let debugDirectory: URL
+ private var stepIndex: Int = 0
+ /// Optional SSH reachability check — returns true if SSH is reachable
+ private let sshCheck: (() async -> Bool)?
+
+ /// Max dimensions to report to Claude API (keeps screenshot payloads small)
+ private static let maxAPIWidth = 1024
+ private static let maxAPIHeight = 768
+
+ static let defaultSystemPrompt = """
+ You are automating the macOS Setup Assistant on a freshly installed virtual machine.
+ Your goal is to complete the setup as quickly as possible with these settings:
+
+ 1. Language: English
+ 2. Country: United States
+ 3. Skip all Apple ID sign-in (look for "Set Up Later", "Skip", "Other Sign-In Options", or similar)
+ 4. Create a local user account with:
+ - Full Name: lume
+ - Account Name: lume
+ - Password: lume
+ 5. Skip or decline all optional features (Siri, Analytics, Screen Time, Location Services, etc.)
+ 6. Accept any privacy/terms screens by clicking Continue/Agree
+ 7. Reach the macOS desktop and enable SSH
+
+ ## Expected screen sequence (adapt as needed — screens may vary by macOS version):
+ 1. "Hello" greeting animation → press Space or click to dismiss
+ 2. Language selection → type "English", select it, press Enter
+ 3. Country/Region → type "United States", press Enter, click Continue
+ 4. Transfer Your Data → click "Set up as new", then Continue
+ 5. Written and Spoken Languages → click Continue
+ 6. Accessibility → click "Not Now"
+ 7. Data & Privacy → click Continue
+ 8. Create a Mac Account → fill Full Name "lume", Tab to skip Account Name (auto-fills), type password "lume", Tab, verify password "lume", Tab to skip Hint, Tab to checkbox, press Space to untoggle "Allow reset with Apple Account", click Continue
+ 9. Apple Account sign-in → look for "Set Up Later" or "Other Sign-In Options" or "Skip". If a "Don't Skip" confirmation appears, Tab to "Skip" and press Enter
+ 10. Terms and Conditions → click "Agree" button (at bottom, not in the text), then click "Agree" again in the confirmation popup
+ 11. Enable Location Services → click Continue, then confirm "Don't Use" in popup
+ 12. Select Your Time Zone → click Continue
+ 13. Analytics → click Continue
+ 14. Screen Time → click "Set Up Later"
+ 15. Siri → click to enable/disable, then click Continue
+ 16. FileVault → click "Not Now", then confirm in popup
+ 17. Choose Your Look → click Continue
+ 18. Update Mac Automatically → click Continue
+ 19. Welcome/Get Started → click "Get Started"
+
+ ## After reaching the desktop, enable SSH:
+ 1. Open Spotlight (Cmd+Space), type "System Settings", press Enter
+ 2. Search for "Remote Login" using Cmd+F, click "Remote Login"
+ 3. Enable the Remote Login toggle, allow all users, click Done
+ 4. Close System Settings (Cmd+Q)
+ 5. The task is now complete.
+
+ ## Important tips:
+ - The screenshot colors may appear tinted/shifted — this is a VNC artifact. Focus on shapes and layout.
+ - After each action, take a screenshot to verify the result before proceeding.
+ - If a button or text is not visible, try waiting 2-3 seconds, then take another screenshot.
+ - Use keyboard shortcuts when buttons are hard to click (Tab to navigate, Enter to confirm, Space to toggle).
+ - If you see a password confirmation field, enter "lume" again.
+ - When you see the macOS desktop with the Dock and menu bar, proceed to enable SSH.
+ - When you see the lock screen with the user "lume", the setup is complete — just stop.
+ """
+
+ init(
+ vncService: VNCService,
+ apiKey: String,
+ model: String = "claude-sonnet-4-6",
+ displayWidth: Int,
+ displayHeight: Int,
+ maxIterations: Int = 200,
+ debug: Bool = false,
+ debugDirectory: URL? = nil,
+ sshCheck: (() async -> Bool)? = nil
+ ) {
+ self.vncService = vncService
+ self.client = AnthropicClient(apiKey: apiKey, model: model)
+ self.vncWidth = displayWidth
+ self.vncHeight = displayHeight
+ // Cap display dimensions for API efficiency
+ self.displayWidth = min(displayWidth, Self.maxAPIWidth)
+ self.displayHeight = min(displayHeight, Self.maxAPIHeight)
+ self.maxIterations = maxIterations
+ self.debug = debug
+ self.debugDirectory = debugDirectory ?? FileManager.default.temporaryDirectory
+ .appendingPathComponent("agent-setup-\(UUID().uuidString)")
+ self.sshCheck = sshCheck
+ }
+
+ /// Run the agent loop to complete Setup Assistant
+ func run(systemPrompt: String? = nil) async throws {
+ let prompt = systemPrompt ?? Self.defaultSystemPrompt
+
+ if debug {
+ try? FileManager.default.createDirectory(at: debugDirectory, withIntermediateDirectories: true)
+ Logger.info("Agent debug screenshots will be saved to \(debugDirectory.path)")
+ }
+
+ // Take initial screenshot
+ Logger.info("Agent taking initial screenshot")
+ let initialScreenshot = try await captureScreenshot()
+
+ // Build initial messages with screenshot
+ var messages: [[String: Any]] = [
+ [
+ "role": "user",
+ "content": [
+ [
+ "type": "text",
+ "text": "Here is the current screen. Please complete the macOS Setup Assistant."
+ ] as [String: Any],
+ [
+ "type": "image",
+ "source": [
+ "type": "base64",
+ "media_type": "image/jpeg",
+ "data": initialScreenshot
+ ]
+ ] as [String: Any]
+ ]
+ ]
+ ]
+
+ var iteration = 0
+ while iteration < maxIterations {
+ iteration += 1
+ Logger.info("Agent iteration \(iteration)/\(maxIterations)")
+
+ // Call Claude API
+ let response = try await client.sendMessage(
+ systemPrompt: prompt,
+ messages: messages,
+ displayWidth: displayWidth,
+ displayHeight: displayHeight
+ )
+
+ // Add assistant response to conversation
+ let assistantContent = try responseToRawContent(response)
+ messages.append(["role": "assistant", "content": assistantContent])
+
+ // Check if Claude is done (no tool use)
+ let toolUseBlocks = response.content.filter {
+ if case .toolUse = $0 { return true }
+ return false
+ }
+
+ if toolUseBlocks.isEmpty {
+ // Claude thinks the task is complete
+ for block in response.content {
+ if case .text(let text) = block {
+ Logger.info("Agent completed: \(text)")
+ }
+ }
+ Logger.info("Agent setup finished after \(iteration) iterations")
+ return
+ }
+
+ // Process each tool use
+ var toolResults: [[String: Any]] = []
+ for block in response.content {
+ guard case .toolUse(let id, let name, let input) = block else { continue }
+ guard name == "computer" else {
+ toolResults.append([
+ "type": "tool_result",
+ "tool_use_id": id,
+ "content": "Unknown tool: \(name)",
+ "is_error": true
+ ])
+ continue
+ }
+
+ let result = try await executeComputerAction(id: id, input: input)
+ toolResults.append(result)
+ }
+
+ // Add tool results to messages
+ messages.append(["role": "user", "content": toolResults])
+
+ // Check SSH reachability — if SSH is up, setup is complete regardless of agent state.
+ // Only start checking after step 50 (setup assistant takes at least that many) and
+ // check every 10 iterations to avoid slowing down the loop.
+ if iteration >= 50 && iteration % 10 == 0, let sshCheck = sshCheck {
+ if await sshCheck() {
+ Logger.info("SSH is reachable — setup complete after \(iteration) iterations")
+ return
+ }
+ }
+
+ // Keep conversation history bounded to prevent request_too_large errors.
+ // Keep the first message (initial screenshot) plus the last 20 messages (10 turns).
+ let maxHistory = 20
+ if messages.count > maxHistory + 1 {
+ let first = messages[0]
+ let recent = Array(messages.suffix(maxHistory))
+ messages = [first] + recent
+ }
+ }
+
+ throw AnthropicError.maxIterationsReached
+ }
+
+ // MARK: - Action Execution
+
+ private func executeComputerAction(id: String, input: [String: AnyCodable]) async throws -> [String: Any] {
+ guard let action = input["action"]?.stringValue else {
+ return errorResult(id: id, message: "Missing 'action' field")
+ }
+
+ Logger.info("Agent action: \(action)", metadata: inputMetadata(input))
+ stepIndex += 1
+
+ do {
+ switch action {
+ case "screenshot":
+ let base64 = try await captureScreenshot()
+ return screenshotResult(id: id, base64: base64)
+
+ case "left_click":
+ guard let coord = coordinateFromInput(input) else {
+ return errorResult(id: id, message: "Missing coordinate for left_click")
+ }
+ try await vncService.sendMouseClick(at: coord, button: .left)
+ try await Task.sleep(nanoseconds: 500_000_000) // 500ms for UI to update
+ let base64 = try await captureScreenshot()
+ return screenshotResult(id: id, base64: base64)
+
+ case "right_click":
+ guard let coord = coordinateFromInput(input) else {
+ return errorResult(id: id, message: "Missing coordinate for right_click")
+ }
+ try await vncService.sendMouseClick(at: coord, button: .right)
+ try await Task.sleep(nanoseconds: 500_000_000)
+ let base64 = try await captureScreenshot()
+ return screenshotResult(id: id, base64: base64)
+
+ case "double_click":
+ guard let coord = coordinateFromInput(input) else {
+ return errorResult(id: id, message: "Missing coordinate for double_click")
+ }
+ try await vncService.sendMouseClick(at: coord, button: .left)
+ try await Task.sleep(nanoseconds: 100_000_000)
+ try await vncService.sendMouseClick(at: coord, button: .left)
+ try await Task.sleep(nanoseconds: 500_000_000)
+ let base64 = try await captureScreenshot()
+ return screenshotResult(id: id, base64: base64)
+
+ case "type":
+ guard let text = input["text"]?.stringValue else {
+ return errorResult(id: id, message: "Missing text for type action")
+ }
+ try await vncService.sendText(text, delayMs: 50)
+ try await Task.sleep(nanoseconds: 300_000_000)
+ let base64 = try await captureScreenshot()
+ return screenshotResult(id: id, base64: base64)
+
+ case "key":
+ guard let keyText = input["text"]?.stringValue else {
+ return errorResult(id: id, message: "Missing text for key action")
+ }
+ try await sendKey(keyText)
+ try await Task.sleep(nanoseconds: 300_000_000)
+ let base64 = try await captureScreenshot()
+ return screenshotResult(id: id, base64: base64)
+
+ case "scroll":
+ // For scroll, we just take a screenshot after — VNC scroll support varies
+ guard let coord = coordinateFromInput(input) else {
+ return errorResult(id: id, message: "Missing coordinate for scroll")
+ }
+ let direction = input["scroll_direction"]?.stringValue ?? "down"
+ let amount = input["scroll_amount"]?.intValue ?? 3
+ try await sendScroll(at: coord, direction: direction, amount: amount)
+ try await Task.sleep(nanoseconds: 500_000_000)
+ let base64 = try await captureScreenshot()
+ return screenshotResult(id: id, base64: base64)
+
+ case "mouse_move":
+ // Just acknowledge — VNC doesn't need explicit moves before clicks
+ return [
+ "type": "tool_result",
+ "tool_use_id": id,
+ "content": "Mouse moved"
+ ]
+
+ case "wait":
+ let duration = input["duration"]?.intValue ?? 2
+ try await Task.sleep(nanoseconds: UInt64(duration) * 1_000_000_000)
+ let base64 = try await captureScreenshot()
+ return screenshotResult(id: id, base64: base64)
+
+ default:
+ return errorResult(id: id, message: "Unsupported action: \(action)")
+ }
+ } catch {
+ Logger.error("Agent action failed", metadata: ["action": action, "error": error.localizedDescription])
+ return errorResult(id: id, message: "Action '\(action)' failed: \(error.localizedDescription)")
+ }
+ }
+
+ // MARK: - Screenshot
+
+ private func captureScreenshot() async throws -> String {
+ let image = try await vncService.captureFramebuffer()
+
+ // Normalize and resize: re-render through a standard RGBA CGContext.
+ // This fixes VNC color channel issues (BGRA→RGBA) and scales to display size.
+ let normalized = normalizeImage(image, maxWidth: displayWidth, maxHeight: displayHeight)
+
+ if debug {
+ saveDebugScreenshot(normalized)
+ }
+
+ // Encode as JPEG for much smaller payloads (PNG screenshots can be 3MB+)
+ guard let base64 = normalized.jpegBase64(quality: 0.7) else {
+ throw UnattendedError.framebufferCaptureFailed("Failed to encode screenshot as JPEG")
+ }
+
+ return base64
+ }
+
+ /// Resize the image to fit within the given bounds. Let CoreGraphics handle pixel format
+ /// conversion naturally — no manual channel swapping needed.
+ private func normalizeImage(_ image: CGImage, maxWidth: Int, maxHeight: Int) -> CGImage {
+ let srcWidth = image.width
+ let srcHeight = image.height
+
+ // Scale to fit within bounds
+ let scale: Double
+ if srcWidth > maxWidth || srcHeight > maxHeight {
+ scale = min(Double(maxWidth) / Double(srcWidth), Double(maxHeight) / Double(srcHeight))
+ } else {
+ scale = 1.0
+ }
+ let newWidth = Int(Double(srcWidth) * scale)
+ let newHeight = Int(Double(srcHeight) * scale)
+
+ let colorSpace = CGColorSpaceCreateDeviceRGB()
+ guard let context = CGContext(
+ data: nil,
+ width: newWidth,
+ height: newHeight,
+ bitsPerComponent: 8,
+ bytesPerRow: 0,
+ space: colorSpace,
+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+ ) else {
+ return image
+ }
+
+ context.interpolationQuality = .high
+ context.draw(image, in: CGRect(x: 0, y: 0, width: newWidth, height: newHeight))
+
+ return context.makeImage() ?? image
+ }
+
+ private func saveDebugScreenshot(_ image: CGImage) {
+ let filename = String(format: "%03d-agent-step.png", stepIndex)
+ let url = debugDirectory.appendingPathComponent(filename)
+ guard let destination = CGImageDestinationCreateWithURL(url as CFURL, "public.png" as CFString, 1, nil) else {
+ return
+ }
+ CGImageDestinationAddImage(destination, image, nil)
+ CGImageDestinationFinalize(destination)
+ Logger.info("Saved debug screenshot", metadata: ["path": url.path])
+ }
+
+ // MARK: - Input Helpers
+
+ /// Parse coordinate from Claude's response and scale to VNC framebuffer space.
+ /// Claude operates in displayWidth x displayHeight space, VNC uses vncWidth x vncHeight.
+ private func coordinateFromInput(_ input: [String: AnyCodable]) -> CGPoint? {
+ if let coordArray = input["coordinate"]?.value as? [Any], coordArray.count == 2 {
+ let x: Double
+ let y: Double
+ if let xi = coordArray[0] as? Int, let yi = coordArray[1] as? Int {
+ x = Double(xi)
+ y = Double(yi)
+ } else if let xd = coordArray[0] as? Double, let yd = coordArray[1] as? Double {
+ x = xd
+ y = yd
+ } else {
+ return nil
+ }
+ // Scale from Claude's coordinate space to VNC framebuffer space
+ let scaleX = Double(vncWidth) / Double(displayWidth)
+ let scaleY = Double(vncHeight) / Double(displayHeight)
+ return CGPoint(x: x * scaleX, y: y * scaleY)
+ }
+ return nil
+ }
+
+ private func sendKey(_ keyText: String) async throws {
+ // Parse key combinations like "ctrl+s", "Return", "space", etc.
+ let parts = keyText.lowercased().split(separator: "+").map(String.init)
+
+ var modifiers: VNCKeyModifiers = .none
+ var mainKey: String = ""
+
+ for part in parts {
+ switch part {
+ case "ctrl", "control": modifiers.insert(.control)
+ case "shift": modifiers.insert(.shift)
+ case "alt", "option": modifiers.insert(.option)
+ case "cmd", "command", "super", "meta": modifiers.insert(.command)
+ default: mainKey = part
+ }
+ }
+
+ let keyCode = keyNameToKeyCode(mainKey)
+ try await vncService.sendKeyPress(keyCode, modifiers: modifiers)
+ }
+
+ private func sendScroll(at point: CGPoint, direction: String, amount: Int) async throws {
+ // VNC scroll is button 4 (up) or button 5 (down)
+ // Use key-based scrolling as fallback (VNC scroll button support varies)
+ for _ in 0..<amount {
+ switch direction {
+ case "up":
+ try await vncService.sendKeyPress(keyNameToKeyCode("up"), modifiers: .none)
+ case "down":
+ try await vncService.sendKeyPress(keyNameToKeyCode("down"), modifiers: .none)
+ case "left":
+ try await vncService.sendKeyPress(keyNameToKeyCode("left"), modifiers: .none)
+ case "right":
+ try await vncService.sendKeyPress(keyNameToKeyCode("right"), modifiers: .none)
+ default:
+ break
+ }
+ try await Task.sleep(nanoseconds: 100_000_000)
+ }
+ }
+
+ private func keyNameToKeyCode(_ name: String) -> UInt16 {
+ // Map common key names to macOS key codes
+ switch name {
+ case "return", "enter": return 36
+ case "tab": return 48
+ case "space": return 49
+ case "escape", "esc": return 53
+ case "backspace", "delete": return 51
+ case "up": return 126
+ case "down": return 125
+ case "left": return 123
+ case "right": return 124
+ case "a": return 0
+ case "b": return 11
+ case "c": return 8
+ case "d": return 2
+ case "e": return 14
+ case "f": return 3
+ case "g": return 5
+ case "h": return 4
+ case "i": return 34
+ case "j": return 38
+ case "k": return 40
+ case "l": return 37
+ case "m": return 46
+ case "n": return 45
+ case "o": return 31
+ case "p": return 35
+ case "q": return 12
+ case "r": return 15
+ case "s": return 1
+ case "t": return 17
+ case "u": return 32
+ case "v": return 9
+ case "w": return 13
+ case "x": return 7
+ case "y": return 16
+ case "z": return 6
+ case "f1": return 122
+ case "f2": return 120
+ case "f3": return 99
+ case "f4": return 118
+ case "f5": return 96
+ case "f6": return 97
+ case "f7": return 98
+ case "f8": return 100
+ case "f9": return 101
+ case "f10": return 109
+ case "f11": return 103
+ case "f12": return 111
+ default: return 49 // space as fallback
+ }
+ }
+
+ // MARK: - Result Builders
+
+ private func screenshotResult(id: String, base64: String) -> [String: Any] {
+ return [
+ "type": "tool_result",
+ "tool_use_id": id,
+ "content": [
+ [
+ "type": "image",
+ "source": [
+ "type": "base64",
+ "media_type": "image/jpeg",
+ "data": base64
+ ]
+ ] as [String: Any]
+ ]
+ ]
+ }
+
+ private func errorResult(id: String, message: String) -> [String: Any] {
+ return [
+ "type": "tool_result",
+ "tool_use_id": id,
+ "content": message,
+ "is_error": true
+ ]
+ }
+
+ private func inputMetadata(_ input: [String: AnyCodable]) -> [String: String] {
+ var meta: [String: String] = [:]
+ if let action = input["action"]?.stringValue { meta["action"] = action }
+ if let text = input["text"]?.stringValue { meta["text"] = text }
+ if let coord = input["coordinate"]?.value as? [Any] {
+ meta["coordinate"] = "\(coord)"
+ }
+ return meta
+ }
+
+ // MARK: - Response Serialization
+
+ private func responseToRawContent(_ response: AnthropicClient.APIResponse) throws -> [[String: Any]] {
+ var content: [[String: Any]] = []
+ for block in response.content {
+ switch block {
+ case .text(let text):
+ content.append(["type": "text", "text": text])
+ case .toolUse(let id, let name, let input):
+ var inputDict: [String: Any] = [:]
+ for (key, val) in input {
+ inputDict[key] = val.value
+ }
+ content.append([
+ "type": "tool_use",
+ "id": id,
+ "name": name,
+ "input": inputDict
+ ])
+ default:
+ break
+ }
+ }
+ return content
+ }
+}
diff --git a/libs/lume/src/Unattended/AnthropicClient.swift b/libs/lume/src/Unattended/AnthropicClient.swift
new file mode 100644
index 0000000000..7296b110a7
--- /dev/null
+++ b/libs/lume/src/Unattended/AnthropicClient.swift
@@ -0,0 +1,240 @@
+import Foundation
+import CoreGraphics
+import ImageIO
+
+/// Minimal Anthropic API client for computer-use agent loop
+struct AnthropicClient {
+ let apiKey: String
+ let model: String
+ let betaFlag: String = "computer-use-2025-11-24"
+
+ private let baseURL = URL(string: "https://api.anthropic.com/v1/messages")!
+
+ struct Message: Codable {
+ let role: String
+ let content: [ContentBlock]
+ }
+
+ enum ContentBlock: Codable {
+ case text(String)
+ case image(base64: String, mediaType: String)
+ case toolUse(id: String, name: String, input: [String: AnyCodable])
+ case toolResult(toolUseId: String, content: [ContentBlock], isError: Bool)
+
+ enum CodingKeys: String, CodingKey {
+ case type, text, source, id, name, input
+ case toolUseId = "tool_use_id"
+ case content, isError = "is_error"
+ }
+
+ func encode(to encoder: Encoder) throws {
+ var container = encoder.container(keyedBy: CodingKeys.self)
+ switch self {
+ case .text(let text):
+ try container.encode("text", forKey: .type)
+ try container.encode(text, forKey: .text)
+ case .image(let base64, let mediaType):
+ try container.encode("image", forKey: .type)
+ let source: [String: String] = [
+ "type": "base64",
+ "media_type": mediaType,
+ "data": base64
+ ]
+ try container.encode(source, forKey: .source)
+ case .toolUse(let id, let name, let input):
+ try container.encode("tool_use", forKey: .type)
+ try container.encode(id, forKey: .id)
+ try container.encode(name, forKey: .name)
+ try container.encode(input, forKey: .input)
+ case .toolResult(let toolUseId, let content, let isError):
+ try container.encode("tool_result", forKey: .type)
+ try container.encode(toolUseId, forKey: .toolUseId)
+ try container.encode(content, forKey: .content)
+ if isError {
+ try container.encode(true, forKey: .isError)
+ }
+ }
+ }
+
+ init(from decoder: Decoder) throws {
+ let container = try decoder.container(keyedBy: CodingKeys.self)
+ let type = try container.decode(String.self, forKey: .type)
+ switch type {
+ case "text":
+ let text = try container.decode(String.self, forKey: .text)
+ self = .text(text)
+ case "tool_use":
+ let id = try container.decode(String.self, forKey: .id)
+ let name = try container.decode(String.self, forKey: .name)
+ let input = try container.decode([String: AnyCodable].self, forKey: .input)
+ self = .toolUse(id: id, name: name, input: input)
+ default:
+ self = .text("[unsupported block type: \(type)]")
+ }
+ }
+ }
+
+ struct APIResponse: Decodable {
+ let id: String
+ let content: [ContentBlock]
+ let stopReason: String?
+
+ enum CodingKeys: String, CodingKey {
+ case id, content
+ case stopReason = "stop_reason"
+ }
+ }
+
+ struct APIError: Decodable {
+ let type: String
+ let error: ErrorDetail
+
+ struct ErrorDetail: Decodable {
+ let type: String
+ let message: String
+ }
+ }
+
+ @MainActor
+ func sendMessage(
+ systemPrompt: String,
+ messages: [[String: Any]],
+ displayWidth: Int,
+ displayHeight: Int
+ ) async throws -> APIResponse {
+ var request = URLRequest(url: baseURL)
+ request.httpMethod = "POST"
+ request.setValue("application/json", forHTTPHeaderField: "content-type")
+ request.setValue(apiKey, forHTTPHeaderField: "x-api-key")
+ request.setValue("2023-06-01", forHTTPHeaderField: "anthropic-version")
+ request.setValue(betaFlag, forHTTPHeaderField: "anthropic-beta")
+ request.timeoutInterval = 120
+
+ let body: [String: Any] = [
+ "model": model,
+ "max_tokens": 4096,
+ "system": systemPrompt,
+ "tools": [
+ [
+ "type": "computer_20251124",
+ "name": "computer",
+ "display_width_px": displayWidth,
+ "display_height_px": displayHeight
+ ] as [String: Any]
+ ],
+ "messages": messages
+ ]
+
+ request.httpBody = try JSONSerialization.data(withJSONObject: body)
+
+ let (data, response) = try await URLSession.shared.data(for: request)
+
+ guard let httpResponse = response as? HTTPURLResponse else {
+ throw AnthropicError.invalidResponse
+ }
+
+ if httpResponse.statusCode != 200 {
+ if let errorResponse = try? JSONDecoder().decode(APIError.self, from: data) {
+ throw AnthropicError.apiError(
+ status: httpResponse.statusCode,
+ message: errorResponse.error.message
+ )
+ }
+ let body = String(data: data, encoding: .utf8) ?? "unknown"
+ throw AnthropicError.apiError(status: httpResponse.statusCode, message: body)
+ }
+
+ return try JSONDecoder().decode(APIResponse.self, from: data)
+ }
+}
+
+enum AnthropicError: Error, LocalizedError {
+ case invalidResponse
+ case apiError(status: Int, message: String)
+ case maxIterationsReached
+
+ var errorDescription: String? {
+ switch self {
+ case .invalidResponse:
+ return "Invalid response from Anthropic API"
+ case .apiError(let status, let message):
+ return "Anthropic API error (\(status)): \(message)"
+ case .maxIterationsReached:
+ return "Agent reached maximum iteration limit"
+ }
+ }
+}
+
+/// Type-erased Codable wrapper for JSON values
+struct AnyCodable: Codable {
+ let value: Any
+
+ init(_ value: Any) {
+ self.value = value
+ }
+
+ init(from decoder: Decoder) throws {
+ let container = try decoder.singleValueContainer()
+ if let intVal = try? container.decode(Int.self) {
+ value = intVal
+ } else if let doubleVal = try? container.decode(Double.self) {
+ value = doubleVal
+ } else if let stringVal = try? container.decode(String.self) {
+ value = stringVal
+ } else if let boolVal = try? container.decode(Bool.self) {
+ value = boolVal
+ } else if let arrayVal = try? container.decode([AnyCodable].self) {
+ value = arrayVal.map { $0.value }
+ } else if let dictVal = try? container.decode([String: AnyCodable].self) {
+ value = dictVal.mapValues { $0.value }
+ } else {
+ value = NSNull()
+ }
+ }
+
+ func encode(to encoder: Encoder) throws {
+ var container = encoder.singleValueContainer()
+ switch value {
+ case let val as Int: try container.encode(val)
+ case let val as Double: try container.encode(val)
+ case let val as String: try container.encode(val)
+ case let val as Bool: try container.encode(val)
+ case is NSNull: try container.encodeNil()
+ default: try container.encodeNil()
+ }
+ }
+
+ var stringValue: String? { value as? String }
+ var intValue: Int? { value as? Int }
+ var doubleValue: Double? { value as? Double }
+ var arrayValue: [Any]? { value as? [Any] }
+}
+
+// MARK: - CGImage to base64
+
+extension CGImage {
+ func pngBase64() -> String? {
+ let mutableData = NSMutableData()
+ guard let destination = CGImageDestinationCreateWithData(mutableData, "public.png" as CFString, 1, nil) else {
+ return nil
+ }
+ CGImageDestinationAddImage(destination, self, nil)
+ guard CGImageDestinationFinalize(destination) else {
+ return nil
+ }
+ return (mutableData as Data).base64EncodedString()
+ }
+
+ func jpegBase64(quality: Double = 0.8) -> String? {
+ let mutableData = NSMutableData()
+ guard let destination = CGImageDestinationCreateWithData(mutableData, "public.jpeg" as CFString, 1, nil) else {
+ return nil
+ }
+ let options: [CFString: Any] = [kCGImageDestinationLossyCompressionQuality: quality]
+ CGImageDestinationAddImage(destination, self, options as CFDictionary)
+ guard CGImageDestinationFinalize(destination) else {
+ return nil
+ }
+ return (mutableData as Data).base64EncodedString()
+ }
+}
diff --git a/libs/lume/src/Unattended/UnattendedInstaller.swift b/libs/lume/src/Unattended/UnattendedInstaller.swift
index 70f5eebc3a..95e842570e 100644
--- a/libs/lume/src/Unattended/UnattendedInstaller.swift
+++ b/libs/lume/src/Unattended/UnattendedInstaller.swift
@@ -5,14 +5,136 @@ import Virtualization
@MainActor
final class UnattendedInstaller {
- /// Run unattended setup on a VM
- /// - Parameters:
- /// - vm: The VM to configure
- /// - config: Unattended configuration with boot commands
- /// - vncPort: VNC port to use (0 for auto-assign)
- /// - noDisplay: If true, don't open the VNC client
- /// - debug: If true, save screenshots with click coordinates
- /// - debugDir: Custom directory for debug screenshots (defaults to unique folder in system temp)
+ /// Run agent-based setup using Claude computer-use API
+ func installWithAgent(
+ vm: VM,
+ apiKey: String,
+ model: String = "claude-sonnet-4-6",
+ maxIterations: Int = 100,
+ systemPrompt: String? = nil,
+ vncPort: Int = 0,
+ noDisplay: Bool = false,
+ debug: Bool = false,
+ debugDir: String? = nil
+ ) async throws {
+ Logger.info("Starting agent-based setup", metadata: [
+ "vm": vm.name,
+ "model": model,
+ "maxIterations": "\(maxIterations)",
+ "debug": "\(debug)"
+ ])
+
+ // Start the VM (same retry logic as preset mode)
+ Logger.info("Starting VM for agent setup (background task)")
+ var vmStartError: Error?
+ let vmTask = Task {
+ var attempts = 0
+ let maxAttempts = 3
+ while attempts < maxAttempts {
+ do {
+ try await vm.run(
+ noDisplay: noDisplay,
+ sharedDirectories: [],
+ mount: nil,
+ vncPort: vncPort,
+ recoveryMode: false,
+ usbMassStoragePaths: []
+ )
+ return
+ } catch {
+ let errorDescription = error.localizedDescription
+ if errorDescription.contains("auxiliary storage") || errorDescription.contains("Failed to lock") {
+ attempts += 1
+ if attempts < maxAttempts {
+ Logger.info("VM start failed due to auxiliary storage lock, retrying", metadata: [
+ "attempt": "\(attempts)/\(maxAttempts)",
+ "waitTime": "5s"
+ ])
+ try await Task.sleep(nanoseconds: 5_000_000_000)
+ continue
+ }
+ }
+ vmStartError = error
+ throw error
+ }
+ }
+ }
+
+ // Give the VM a moment to start
+ try await Task.sleep(nanoseconds: 2_000_000_000)
+
+ if let error = vmStartError {
+ Logger.error("VM failed to start", metadata: ["error": error.localizedDescription])
+ throw error
+ }
+
+ // Wait for boot (agent mode uses a fixed 60s boot wait)
+ let bootWait: UInt64 = 60
+ Logger.info("Waiting for VM to boot", metadata: ["bootWait": "\(bootWait)s"])
+ try await Task.sleep(nanoseconds: bootWait * 1_000_000_000)
+
+ // Connect VNC input client
+ Logger.info("Connecting VNC input client for agent automation")
+ try await vm.vncService.connectInputClient()
+
+ // Get display dimensions from VNC framebuffer
+ let framebufferSize = await vm.vncService.getFrameBufferSize() ?? (width: 1920, height: 1440)
+ let displayWidth = Int(framebufferSize.width)
+ let displayHeight = Int(framebufferSize.height)
+
+ Logger.info("VNC framebuffer size", metadata: [
+ "width": "\(displayWidth)",
+ "height": "\(displayHeight)"
+ ])
+
+ // Create SSH reachability check closure
+ let macAddress = vm.vmDirContext.config.macAddress
+ let sshCheck: (() async -> Bool)? = macAddress != nil ? {
+ guard let ip = DHCPLeaseParser.getIPAddress(forMAC: macAddress!) else {
+ return false
+ }
+ Logger.info("Checking SSH reachability", metadata: ["ip": ip])
+ let runner = HealthCheckRunner()
+ let check = HealthCheck(type: "ssh", user: "lume", password: "lume", timeout: 5, retries: 1, retryDelay: 0)
+ return (try? await runner.run(check: check, vmIP: ip)) ?? false
+ } : nil
+
+ // Create and run the agent
+ let debugDirectory: URL? = debugDir.map { URL(fileURLWithPath: $0) }
+ let agent = AgentSetupRunner(
+ vncService: vm.vncService,
+ apiKey: apiKey,
+ model: model,
+ displayWidth: displayWidth,
+ displayHeight: displayHeight,
+ maxIterations: maxIterations,
+ debug: debug,
+ debugDirectory: debugDirectory,
+ sshCheck: sshCheck
+ )
+
+ do {
+ try await agent.run(systemPrompt: systemPrompt)
+ Logger.info("Agent setup completed successfully")
+
+ vm.vncService.disconnectInputClient()
+
+ // Stop the VM
+ Logger.info("Agent setup finished - stopping VM")
+ vmTask.cancel()
+ try? await vm.stop()
+ Logger.info("VM stopped", metadata: ["name": vm.name])
+ } catch {
+ Logger.error("Agent setup failed", metadata: ["error": error.localizedDescription])
+ vm.vncService.disconnectInputClient()
+ vmTask.cancel()
+ try? await vm.stop()
+ Logger.info("VM stopped after failure", metadata: ["name": vm.name])
+ throw error
+ }
+ }
+
+ /// Run unattended setup on a VM using preset commands
func install(
vm: VM,
config: UnattendedConfig,