Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(lume): add configurable delay parameter to type command (#982)
Francesco Bonacci52f8e783888d
diff --git a/libs/lume/src/Resources/unattended-presets/sequoia.yml b/libs/lume/src/Resources/unattended-presets/sequoia.ymlindex cef271c735..680b4ec9f0 100644--- a/libs/lume/src/Resources/unattended-presets/sequoia.yml+++ b/libs/lume/src/Resources/unattended-presets/sequoia.yml@@ -200,11 +200,11 @@ boot_commands:# ============================================- "<cmd+space>"- "<delay 5>"- - "<type 'Terminal'>"+ - "<type 'Terminal', delay=350>"- "<delay 5>"- "<enter>"- "<delay 5>"- - "<type 'defaults write NSGlobalDomain AppleKeyboardUIMode -int 3'>"+ - "<type 'defaults write NSGlobalDomain AppleKeyboardUIMode -int 3', delay=350>"- "<enter>"- "<delay 3>"- "<cmd+q>"diff --git a/libs/lume/src/Resources/unattended-presets/tahoe.yml b/libs/lume/src/Resources/unattended-presets/tahoe.ymlindex 2b5d0da12c..fda8707633 100644--- a/libs/lume/src/Resources/unattended-presets/tahoe.yml+++ b/libs/lume/src/Resources/unattended-presets/tahoe.yml@@ -211,11 +211,11 @@ boot_commands:# ============================================- "<cmd+space>"- "<delay 5>"- - "<type 'Terminal'>"+ - "<type 'Terminal', delay=350>"- "<delay 5>"- "<enter>"- "<delay 5>"- - "<type 'defaults write NSGlobalDomain AppleKeyboardUIMode -int 3'>"+ - "<type 'defaults write NSGlobalDomain AppleKeyboardUIMode -int 3', delay=350>"- "<enter>"- "<delay 3>"- "<cmd+q>"diff --git a/libs/lume/src/Unattended/BootCommandParser.swift b/libs/lume/src/Unattended/BootCommandParser.swiftindex 67379dafd6..64d087b0c4 100644--- a/libs/lume/src/Unattended/BootCommandParser.swift+++ b/libs/lume/src/Unattended/BootCommandParser.swift@@ -17,8 +17,8 @@ enum BootCommand: Sendable {/// Click at specific screen coordinatescase clickAt(x: Int, y: Int)- /// Type text string- case typeText(String)+ /// Type text string with optional delay between characters (in milliseconds)+ case typeText(String, delay: Int?)/// Press a special keycase keyPress(SpecialKey)@@ -239,8 +239,9 @@ struct BootCommandParser {}private func parseTypeCommand(_ input: String) throws -> BootCommand? {- // Match <type 'text'> or <type "text">- let pattern = #"^<type\s+['"](.*)['"]>$"#+ // Match <type 'text'> or <type "text"> with optional delay parameter+ // Examples: <type 'hello'>, <type 'hello', delay=100>+ let pattern = #"^<type\s+['"](.*)['"](?:\s*,\s*delay\s*=\s*(\d+))?\s*>$"#guard let regex = try? NSRegularExpression(pattern: pattern, options: []),let match = regex.firstMatch(in: input, range: NSRange(input.startIndex..., in: input)),let textRange = Range(match.range(at: 1), in: input)@@ -249,7 +250,15 @@ struct BootCommandParser {}let text = String(input[textRange])- return .typeText(text)+ var delay: Int? = nil++ if match.numberOfRanges > 2,+ let delayRange = Range(match.range(at: 2), in: input),+ let delayValue = Int(input[delayRange]) {+ delay = delayValue+ }++ return .typeText(text, delay: delay)}private func parseDelayCommand(_ input: String) throws -> BootCommand? {diff --git a/libs/lume/src/Unattended/VNCAutomation.swift b/libs/lume/src/Unattended/VNCAutomation.swiftindex 46fc2c1baa..8fe07c78e5 100644--- a/libs/lume/src/Unattended/VNCAutomation.swift+++ b/libs/lume/src/Unattended/VNCAutomation.swift@@ -64,8 +64,8 @@ final class VNCAutomation {button: .left)- case .typeText(let text):- try await vncService.sendText(text)+ case .typeText(let text, let delay):+ try await vncService.sendText(text, delayMs: delay)case .keyPress(let key):let keyCode = specialKeyToKeyCode(key)@@ -320,8 +320,11 @@ final class VNCAutomation {return "click '\(text)' (index: \(index), xoffset: \(xOffset), yoffset: \(yOffset))"case .clickAt(let x, let y):return "click at (\(x), \(y))"- case .typeText(let text):+ case .typeText(let text, let delay):let displayText = text.count > 20 ? String(text.prefix(20)) + "..." : text+ if let delay = delay {+ return "type '\(displayText)' (delay: \(delay)ms)"+ }return "type '\(displayText)'"case .keyPress(let key):return "press <\(key.rawValue)>"diff --git a/libs/lume/src/VNC/VNCService.swift b/libs/lume/src/VNC/VNCService.swiftindex cb729d2817..719f70acf6 100644--- a/libs/lume/src/VNC/VNCService.swift+++ b/libs/lume/src/VNC/VNCService.swift@@ -17,7 +17,7 @@ protocol VNCService {func captureFramebuffer() async throws -> CGImagefunc sendMouseClick(at point: CGPoint, button: VNCMouseButton) async throwsfunc sendKeyPress(_ keyCode: UInt16, modifiers: VNCKeyModifiers) async throws- func sendText(_ text: String) async throws+ func sendText(_ text: String, delayMs: Int?) async throwsfunc sendCharWithModifiers(_ char: Character, modifiers: VNCKeyModifiers) async throws// VNC client for input@@ -330,11 +330,14 @@ final class DefaultVNCService: VNCService {Logger.debug("Key press sent via VNC client")}- func sendText(_ text: String) async throws {+ func sendText(_ text: String, delayMs: Int? = nil) async throws {guard let client = vncClient else {throw UnattendedError.inputSimulationFailed("VNC input client not connected")}+ // Use provided delay or default to 50ms+ let delayNs = UInt64(delayMs ?? 50) * 1_000_000+// Type each character using X11 keysymsfor char in text {let (keysym, needsShift) = charToKeysym(char)@@ -344,14 +347,14 @@ final class DefaultVNCService: VNCService {}try await client.sendKeyEvent(key: keysym, down: true)- try await Task.sleep(nanoseconds: 100_000_000) // 100ms key hold+ try await Task.sleep(nanoseconds: delayNs) // key holdtry await client.sendKeyEvent(key: keysym, down: false)if needsShift {try await client.sendKeyEvent(key: X11Keysym.shiftL.rawValue, down: false)}- try await Task.sleep(nanoseconds: 250_000_000) // 250ms between characters+ try await Task.sleep(nanoseconds: delayNs) // between characters}}