Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(lume): add configurable delay parameter to type command (#982)

Francesco Bonacci52f8e783888d
diff --git a/libs/lume/src/Resources/unattended-presets/sequoia.yml b/libs/lume/src/Resources/unattended-presets/sequoia.yml
index cef271c735..680b4ec9f0 100644
--- a/libs/lume/src/Resources/unattended-presets/sequoia.yml
+++ b/libs/lume/src/Resources/unattended-presets/sequoia.yml
@@ -200,11 +200,11 @@ boot_commands:
# ============================================
- "<cmd+space>"
- "<delay 5>"
- - "<type 'Terminal'>"
+ - "<type 'Terminal', delay=350>"
- "<delay 5>"
- "<enter>"
- "<delay 5>"
- - "<type 'defaults write NSGlobalDomain AppleKeyboardUIMode -int 3'>"
+ - "<type 'defaults write NSGlobalDomain AppleKeyboardUIMode -int 3', delay=350>"
- "<enter>"
- "<delay 3>"
- "<cmd+q>"
diff --git a/libs/lume/src/Resources/unattended-presets/tahoe.yml b/libs/lume/src/Resources/unattended-presets/tahoe.yml
index 2b5d0da12c..fda8707633 100644
--- a/libs/lume/src/Resources/unattended-presets/tahoe.yml
+++ b/libs/lume/src/Resources/unattended-presets/tahoe.yml
@@ -211,11 +211,11 @@ boot_commands:
# ============================================
- "<cmd+space>"
- "<delay 5>"
- - "<type 'Terminal'>"
+ - "<type 'Terminal', delay=350>"
- "<delay 5>"
- "<enter>"
- "<delay 5>"
- - "<type 'defaults write NSGlobalDomain AppleKeyboardUIMode -int 3'>"
+ - "<type 'defaults write NSGlobalDomain AppleKeyboardUIMode -int 3', delay=350>"
- "<enter>"
- "<delay 3>"
- "<cmd+q>"
diff --git a/libs/lume/src/Unattended/BootCommandParser.swift b/libs/lume/src/Unattended/BootCommandParser.swift
index 67379dafd6..64d087b0c4 100644
--- a/libs/lume/src/Unattended/BootCommandParser.swift
+++ b/libs/lume/src/Unattended/BootCommandParser.swift
@@ -17,8 +17,8 @@ enum BootCommand: Sendable {
/// Click at specific screen coordinates
case clickAt(x: Int, y: Int)
- /// Type text string
- case typeText(String)
+ /// Type text string with optional delay between characters (in milliseconds)
+ case typeText(String, delay: Int?)
/// Press a special key
case keyPress(SpecialKey)
@@ -239,8 +239,9 @@ struct BootCommandParser {
}
private func parseTypeCommand(_ input: String) throws -> BootCommand? {
- // Match <type 'text'> or <type "text">
- let pattern = #"^<type\s+['"](.*)['"]>$"#
+ // Match <type 'text'> or <type "text"> with optional delay parameter
+ // Examples: <type 'hello'>, <type 'hello', delay=100>
+ let pattern = #"^<type\s+['"](.*)['"](?:\s*,\s*delay\s*=\s*(\d+))?\s*>$"#
guard let regex = try? NSRegularExpression(pattern: pattern, options: []),
let match = regex.firstMatch(in: input, range: NSRange(input.startIndex..., in: input)),
let textRange = Range(match.range(at: 1), in: input)
@@ -249,7 +250,15 @@ struct BootCommandParser {
}
let text = String(input[textRange])
- return .typeText(text)
+ var delay: Int? = nil
+
+ if match.numberOfRanges > 2,
+ let delayRange = Range(match.range(at: 2), in: input),
+ let delayValue = Int(input[delayRange]) {
+ delay = delayValue
+ }
+
+ return .typeText(text, delay: delay)
}
private func parseDelayCommand(_ input: String) throws -> BootCommand? {
diff --git a/libs/lume/src/Unattended/VNCAutomation.swift b/libs/lume/src/Unattended/VNCAutomation.swift
index 46fc2c1baa..8fe07c78e5 100644
--- a/libs/lume/src/Unattended/VNCAutomation.swift
+++ b/libs/lume/src/Unattended/VNCAutomation.swift
@@ -64,8 +64,8 @@ final class VNCAutomation {
button: .left
)
- case .typeText(let text):
- try await vncService.sendText(text)
+ case .typeText(let text, let delay):
+ try await vncService.sendText(text, delayMs: delay)
case .keyPress(let key):
let keyCode = specialKeyToKeyCode(key)
@@ -320,8 +320,11 @@ final class VNCAutomation {
return "click '\(text)' (index: \(index), xoffset: \(xOffset), yoffset: \(yOffset))"
case .clickAt(let x, let y):
return "click at (\(x), \(y))"
- case .typeText(let text):
+ case .typeText(let text, let delay):
let displayText = text.count > 20 ? String(text.prefix(20)) + "..." : text
+ if let delay = delay {
+ return "type '\(displayText)' (delay: \(delay)ms)"
+ }
return "type '\(displayText)'"
case .keyPress(let key):
return "press <\(key.rawValue)>"
diff --git a/libs/lume/src/VNC/VNCService.swift b/libs/lume/src/VNC/VNCService.swift
index cb729d2817..719f70acf6 100644
--- a/libs/lume/src/VNC/VNCService.swift
+++ b/libs/lume/src/VNC/VNCService.swift
@@ -17,7 +17,7 @@ protocol VNCService {
func captureFramebuffer() async throws -> CGImage
func sendMouseClick(at point: CGPoint, button: VNCMouseButton) async throws
func sendKeyPress(_ keyCode: UInt16, modifiers: VNCKeyModifiers) async throws
- func sendText(_ text: String) async throws
+ func sendText(_ text: String, delayMs: Int?) async throws
func sendCharWithModifiers(_ char: Character, modifiers: VNCKeyModifiers) async throws
// VNC client for input
@@ -330,11 +330,14 @@ final class DefaultVNCService: VNCService {
Logger.debug("Key press sent via VNC client")
}
- func sendText(_ text: String) async throws {
+ func sendText(_ text: String, delayMs: Int? = nil) async throws {
guard let client = vncClient else {
throw UnattendedError.inputSimulationFailed("VNC input client not connected")
}
+ // Use provided delay or default to 50ms
+ let delayNs = UInt64(delayMs ?? 50) * 1_000_000
+
// Type each character using X11 keysyms
for char in text {
let (keysym, needsShift) = charToKeysym(char)
@@ -344,14 +347,14 @@ final class DefaultVNCService: VNCService {
}
try await client.sendKeyEvent(key: keysym, down: true)
- try await Task.sleep(nanoseconds: 100_000_000) // 100ms key hold
+ try await Task.sleep(nanoseconds: delayNs) // key hold
try await client.sendKeyEvent(key: keysym, down: false)
if needsShift {
try await client.sendKeyEvent(key: X11Keysym.shiftL.rawValue, down: false)
}
- try await Task.sleep(nanoseconds: 250_000_000) // 250ms between characters
+ try await Task.sleep(nanoseconds: delayNs) // between characters
}
}