Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
cua-driver: overlay z-ordering, background focus fixes, hermes form-fill tests (#1375)
ddupontd010242355cf
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swiftindex e9398f9106..937ee984cd 100644--- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift+++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift@@ -207,12 +207,18 @@ public final class AgentCursor {/// the overlay is hidden or the cursor disabled.private var activationObserver: NSObjectProtocol?- /// Short-lived task that re-runs `pinAbove` a few times after- /// each click to catch the async window-level raise that the- /// target does without the app becoming frontmost (so the- /// activation observer misses it). Cancelled and respawned on- /// every `pinAbove` call.- private var defensiveRepinTask: Task<Void, Never>?+ /// Continuous repin loop: fires at ~30 fps while the overlay is+ /// z-pinned to a target. Catches async window-level raises that+ /// macOS issues after AX actions (the target can briefly elevate+ /// above the overlay before any one-shot tick fires). Replaces+ /// the previous sparse [60, 180, 360 …] defensive-repin schedule+ /// — at 33ms the overlay snaps back within one frame rather than+ /// up to 300ms, making the "dip behind" artefact imperceptible.+ ///+ /// `CGWindowListCopyWindowInfo` at 30 fps costs a few hundred+ /// microseconds per call — well within the 33ms budget and much+ /// cheaper than pixel capture.+ private var continuousRepinTask: Task<Void, Never>?/// Count of consecutive `reapplyPinAbove` ticks that couldn't/// find the pinned pid's on-screen window. Used to tolerate@@ -294,14 +300,11 @@ public final class AgentCursor {}/// Pin the overlay just above the given pid's frontmost on-screen- /// window. Keeps the overlay at `.floating` (the init default)- /// and orders it above the target window so consecutive clicks on- /// the same target skip redundant re-orders. See- /// `reapplyPinAbove()` for the rationale on staying at `.floating`- /// instead of demoting to `.normal` — short version: `.normal`- /// lets Electron / Chromium targets transiently push themselves- /// above the cursor mid-click, making the overlay read as "blinks- /// out for a frame" on every click.+ /// window at `.normal` level. `order(.above, relativeTo:)` places+ /// the overlay exactly one slot above the target so any windows+ /// that were already above the target remain above the overlay —+ /// producing `[target, overlay, fg-windows]` z-ordering. Consecutive+ /// clicks on the same target skip redundant re-orders.////// When the target has no on-screen window (hidden launch still/// pending, offscreen window, etc.) the overlay is ordered out@@ -313,31 +316,27 @@ public final class AgentCursor {missedPinCount = 0 // fresh pin — any earlier miss streak is staleensureActivationObserver()reapplyPinAbove()- scheduleDefensiveRepin()+ startContinuousRepin()}- /// Re-run `pinAbove` a few times over the next ~1200ms to catch- /// the async window-level raise macOS sometimes does a few- /// frames after an AX click — when the target's *window* rises- /// in z-order but the *app* doesn't become frontmost, so- /// `didActivateApplicationNotification` never fires. Each call- /// is idempotent when the overlay is already correctly pinned;- /// cheap enough to run on a short schedule.+ /// Start a continuous ~30 fps repin loop for the current target.+ /// Fires every 33ms while `pinnedPid` is set, re-ordering the+ /// overlay just above the target each tick. This catches window-+ /// level raises that macOS issues asynchronously after AX actions+ /// (without the activation notification firing) — the overlay+ /// snaps back within ≤33ms instead of waiting up to 300ms for a+ /// sparse defensive tick.///- /// Coverage must span the full click lifecycle — `playClickPress`- /// runs for 650ms and `finishClick`'s dwell adds another- /// ~250ms. An earlier 700ms schedule let late-arriving target- /// raises (Electron / redraw-heavy apps in particular) land- /// after the last tick, stranding the overlay under the target- /// for the rest of the ripple. The current schedule tails out- /// to 1200ms with buffer, so ticks keep firing through the- /// ripple + dwell even for slower targets.- private func scheduleDefensiveRepin() {- defensiveRepinTask?.cancel()- defensiveRepinTask = Task { @MainActor [weak self] in- for delayMs in [60, 180, 360, 600, 900, 1200] {- try? await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000)- guard let self, !Task.isCancelled else { return }+ /// The loop exits automatically when `pinnedPid` is cleared+ /// (overlay hidden or target gone) so there's no separate+ /// cancellation needed in the common path. Explicit cancellation+ /// is still done in `hide()` for immediate tear-down.+ private func startContinuousRepin() {+ continuousRepinTask?.cancel()+ continuousRepinTask = Task { @MainActor [weak self] in+ while !Task.isCancelled {+ try? await Task.sleep(nanoseconds: 33_000_000) // ~30 fps+ guard let self, !Task.isCancelled, self.pinnedPid != nil else { return }self.reapplyPinAbove()}}@@ -378,24 +377,20 @@ public final class AgentCursor {return}missedPinCount = 0- // Keep the overlay at its initial `.floating` level — above- // ordinary `.normal` app windows without competing for z-order- // against them. Previously this demoted the overlay to- // `.normal` and re-ordered it just above the target so- // unrelated apps stacked over the target would occlude the- // cursor (aesthetic nicety). But at `.normal`, any redraw /- // re-raise on the target — especially for Electron / Chromium- // apps that re-stack their own window on AX-dispatched clicks- // — can transiently push the target above the overlay. Ripple- // animation then plays behind the app and the cursor reads- // as "disappears for a second" on every click. At `.floating`,- // the window server guarantees ordering against `.normal`, so- // the cursor stays visible through the click regardless of- // what the target does to its own z-stack.+ // Place the overlay just above the target within the `.normal`+ // window level — producing the ordering:+ // [target-window, overlay, windows-that-were-above-target]+ // This means windows that were already in front of the target+ // (e.g. the user's foreground app) correctly occlude the cursor,+ // making it clear which window the agent is interacting with.//- // `order(.above, relativeTo:)` is still worth running: it- // keeps the overlay above other `.floating` windows (ours or- // the system's) that might otherwise appear over it.+ // The overlay was previously kept at `.floating` so the window+ // server guaranteed it stayed above all `.normal` windows. The+ // downside is that it covered the user's foreground app, making+ // the z-ordering visually misleading for background-window+ // automation. Now at `.normal`, apps above the target remain+ // above the overlay — background interaction stays visually+ // sandwiched where it belongs.win.order(.above, relativeTo: targetWindow.id)pinnedWindowId = targetWindow.id}@@ -432,26 +427,26 @@ public final class AgentCursor {NSWorkspace.shared.notificationCenter.removeObserver(obs)activationObserver = nil}- defensiveRepinTask?.cancel()- defensiveRepinTask = nil+ continuousRepinTask?.cancel()+ continuousRepinTask = nil}/// Hide the overlay window. No-op if not shown. Keeps the window/// retained so the next `show()` is instant.///- /// Also clears the pin state and cancels the defensive repin- /// task. Without this cleanup, subsequent- /// `didActivateApplicationNotification` events (or any still-- /// queued defensive repin ticks) would call `reapplyPinAbove`,- /// which re-orders the overlay window into the z-stack — visibly- /// resurrecting the cursor the idle-hide timer just removed.+ /// Also clears the pin state and stops the continuous repin loop.+ /// Without this cleanup, the loop would keep calling+ /// `reapplyPinAbove`, re-ordering the overlay window back into+ /// the z-stack — visibly resurrecting the cursor the idle-hide+ /// timer just removed.public func hide() {overlay?.orderOut(nil)pinnedWindowId = nilpinnedPid = nilmissedPinCount = 0- defensiveRepinTask?.cancel()- defensiveRepinTask = nil+ continuousRepinTask?.cancel()+ continuousRepinTask = nil+ clearFocusRect()}/// Move the cursor immediately to a screen-point coordinate. No@@ -545,6 +540,20 @@ public final class AgentCursor {scheduleIdleHide()}+ /// Show a glowing highlight rectangle around the given screen rect.+ /// Used by ClickTool to draw a focus indicator on the targeted AX+ /// element. Pass nil to clear the rect. No-op when disabled.+ public func showFocusRect(_ rect: CGRect?) {+ guard isEnabled else { return }+ AgentCursorRenderer.shared.focusRect = rect+ }++ /// Clear the glowing focus rect. Called by hide() so the rect+ /// doesn't linger after the cursor auto-hides.+ private func clearFocusRect() {+ AgentCursorRenderer.shared.focusRect = nil+ }+/// For tests + spike code: tear down the window so the next/// `show()` rebuilds it from scratch. Not part of the public/// tool-surface contract.diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorOverlayWindow.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorOverlayWindow.swiftindex af97941b14..b09c43ae6d 100644--- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorOverlayWindow.swift+++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorOverlayWindow.swift@@ -36,14 +36,13 @@ public final class AgentCursorOverlayWindow: NSWindow {backgroundColor = .clearhasShadow = falseignoresMouseEvents = true- // Start at `.floating` — above ordinary app windows, below- // menus and modals. `AgentCursor.pinAbove(pid:)` re-parents- // the overlay into `.normal` and z-orders it just above the- // target's frontmost window on every click, so unrelated apps- // layered over the target correctly occlude the cursor. The- // `.floating` default only matters for the initial show- // before any pointer action has run.- level = .floating+ // `.normal` level so the overlay is sandwiched in the regular+ // window stack. `AgentCursor.pinAbove(pid:)` calls+ // `order(.above, relativeTo: targetWindowId)` to place the+ // overlay exactly one slot above the target window — windows+ // that were already above the target remain above the overlay.+ // This produces the ordering: [target, overlay, fg-windows].+ level = .normalcollectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary,]diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swiftindex dba27b7a19..453bf6a50c 100644--- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift+++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift@@ -51,6 +51,10 @@ public final class AgentCursorRenderer {private(set) public var position: CGPoint = .init(x: -200, y: -200)/// Visual heading in radians (tip points along this vector).private(set) public var heading: Double = .pi / 4 // ~NW, like the OS cursor+ /// Screen-space bounding rect of the most recently targeted AX element.+ /// Drawn as a glowing highlight rectangle in the overlay view.+ /// Nil when no element is targeted or the cursor is hidden.+ public var focusRect: CGRect? = nil// -------- Internal state ---------------------------------------------diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swiftindex 797b5f7c8a..e2dbf13e2d 100644--- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift+++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift@@ -17,8 +17,9 @@ public struct AgentCursorView: View {public var body: some View {TimelineView(.animation(minimumInterval: 1.0 / 120.0)) { ctx in- Canvas { gctx, _ in+ Canvas { gctx, size inrenderer.tick(now: ctx.date.timeIntervalSinceReferenceDate)+ drawFocusRect(in: gctx, canvasSize: size)drawCursor(in: gctx)}.ignoresSafeArea()@@ -26,6 +27,53 @@ public struct AgentCursorView: View {}}+ /// Draw a glowing highlight rectangle around the currently targeted AX+ /// element (if `renderer.focusRect` is set). The rect is in screen-point+ /// coordinates; the overlay window's frame is the full screen, so we+ /// convert from screen-point to canvas-local by subtracting the screen's+ /// origin. Uses a cyan-glow border with a faint fill to mark the target.+ private func drawFocusRect(in ctx: GraphicsContext, canvasSize: CGSize) {+ guard let screenRect = renderer.focusRect else { return }+ // The overlay window covers the whole screen, and the canvas's+ // coordinate origin is the top-left of the screen. Screen-point+ // coordinates (CoreGraphics, top-left origin on macOS) map+ // directly to canvas coordinates — no offset needed.+ let r = CGRect(+ x: screenRect.minX,+ y: screenRect.minY,+ width: screenRect.width,+ height: screenRect.height+ )+ let cornerRadius: CGFloat = 4+ let rounded = Path(roundedRect: r, cornerRadius: cornerRadius)++ // Faint fill — shows the full extent of the target.+ ctx.fill(+ rounded,+ with: .color(+ Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.08)+ )+ )++ // Solid bright border.+ ctx.stroke(+ rounded,+ with: .color(+ Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.90)+ ),+ lineWidth: 2+ )++ // Wide soft glow stroke for the neon halo effect.+ ctx.stroke(+ rounded,+ with: .color(+ Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.30)+ ),+ lineWidth: 8+ )+ }+/// Draw the cursor arrow centered on `renderer.position`, rotated to/// `renderer.heading`. The shape is a 4-vertex pointer arrow with/// the tip along +x before rotation, which the caller rotates bydiff --git a/libs/cua-driver/Sources/CuaDriverCore/Input/AXInput.swift b/libs/cua-driver/Sources/CuaDriverCore/Input/AXInput.swiftindex e93333565e..4dc764ba7f 100644--- a/libs/cua-driver/Sources/CuaDriverCore/Input/AXInput.swift+++ b/libs/cua-driver/Sources/CuaDriverCore/Input/AXInput.swift@@ -142,6 +142,30 @@ public enum AXInput {)}+ /// The element's screen-point bounding rect (top-left origin).+ /// Returns nil when the element has no position / size.+ /// Public entry point used by ClickTool for the focus-rect overlay.+ public static func screenBoundingRect(of element: AXUIElement) -> CGRect? {+ return boundingRect(of: element)+ }++ /// Read the element's `AXChildren` attribute. Returns an empty array on+ /// any error (element has no children, AX permission denied, etc.).+ public static func children(of element: AXUIElement) -> [AXUIElement] {+ var ref: CFTypeRef?+ guard+ AXUIElementCopyAttributeValue(element, "AXChildren" as CFString, &ref) == .success,+ let arr = ref as? [AXUIElement]+ else { return [] }+ return arr+ }++ /// Read a string attribute from an element. Returns nil when the attribute+ /// is absent, unreadable, or not a string.+ public static func stringAttribute(_ name: String, of element: AXUIElement) -> String? {+ return attributeString(element, name)+ }+/// The element's on-screen center in screen-point coordinates/// (top-left origin). Returns nil when the element has no/// position / size (menus, offscreen elements, hidden rows).diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swiftindex 900f0a3004..1e6ac10446 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift@@ -269,6 +269,24 @@ public enum ClickTool {) {try AXInput.performAction(axAction, on: element)}+ // For text fields (AXTextField / AXTextArea), WebKit establishes+ // DOM focus asynchronously after AXPress returns. Without a pause,+ // a follow-up type_text_chars call races with WebKit's focus setup+ // and sends chars before the input is active — chars are silently+ // dropped. This is especially pronounced for email/number inputs+ // when the app is backgrounded (Safari is non-frontmost).+ //+ // Empirically, 800 ms is reliably sufficient: a direct Python+ // integration test showed immediate click+type fails but click ++ // 2 s + type succeeds; 800 ms gives comfortable margin while+ // keeping the UX fast enough for interactive use.+ let target = AXInput.describe(element)+ if axAction == "AXPress",+ let role = target.role,+ role == "AXTextField" || role == "AXTextArea"+ {+ try? await Task.sleep(for: .milliseconds(800))+ }// AX-dispatched clicks can raise the target window to// the top of its level, leaving the overlay stranded// beneath it. Re-pin BEFORE the press-in / dwell so@@ -279,6 +297,14 @@ public enum ClickTool {await MainActor.run {AgentCursor.shared.pinAbove(pid: pid)}+ // If the element has a bounding rect, show a glowing focus+ // highlight on the cursor overlay so the user can see which+ // element the agent is targeting.+ if let rect = AXInput.screenBoundingRect(of: element) {+ await MainActor.run {+ AgentCursor.shared.showFocusRect(rect)+ }+ }// Press-in / release pulse on the cursor — purely// visual confirmation that the click fired.// No-op when disabled.@@ -287,9 +313,35 @@ public enum ClickTool {// period and arm the idle-hide timer. No-op when// disabled.await AgentCursor.shared.finishClick(pid: pid)- let target = AXInput.describe(element)var summary ="✅ Performed \(axAction) on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"."+ // For popup buttons (HTML <select> elements in Safari/WebKit):+ // the native macOS popup menu that AXPress opens immediately+ // closes when the app is non-frontmost, so the visible options+ // are never selectable from the keyboard. Instead, list the+ // available options from the AX children and direct the caller+ // to use set_value — which AX-presses the specific child option+ // directly, bypassing the native menu entirely.+ if target.role == "AXPopUpButton" {+ let children = AXInput.children(of: element)+ let options: [(title: String, value: String)] = children.compactMap { child in+ let t = AXInput.stringAttribute("AXTitle", of: child) ?? ""+ let v = AXInput.stringAttribute("AXValue", of: child) ?? ""+ guard !t.isEmpty || !v.isEmpty else { return nil }+ return (title: t, value: v)+ }+ if !options.isEmpty {+ let optList = options.map { o in+ o.value.isEmpty || o.value == o.title ? "\"\(o.title)\"" : "\"\(o.title)\" (value: \(o.value))"+ }.joined(separator: ", ")+ summary += "\n\n⚠️ This is a popup/select button. The native macOS menu"+ summary += " closes immediately when the window is in the background."+ summary += " Do NOT use click again — instead, use:"+ summary += "\n set_value(pid: \(pid), window_id: \(windowId), element_index: \(index),"+ summary += " value: \"<option title>\")"+ summary += "\nAvailable options: [\(optList)]"+ }+ }// If the element didn't advertise the action we just// dispatched, append a non-fatal warning. The AX call// returned `.success` but the element almost certainlydiff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swiftindex 40b548327a..b6c55ec801 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift@@ -233,8 +233,17 @@ public enum LaunchAppTool {summary += "\n→ Call get_window_state(pid: \(info.pid), window_id) to inspect."}+ let launchResult = LaunchResult(info: info, windows: windowRecords)+ let textContent: Tool.Content = .text(+ text: summary, annotations: nil, _meta: nil)+ if let result = try? CallTool.Result(+ content: [textContent],+ structuredContent: launchResult+ ) {+ return result+ }return CallTool.Result(- content: [.text(text: summary, annotations: nil, _meta: nil)]+ content: [textContent])} catch let error as AppLauncher.LaunchError {return errorResult(error.description)diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swiftindex 73c82926e3..e8b6e95527 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift@@ -125,14 +125,19 @@ public enum ListWindowsTool {)}- return CallTool.Result(- content: [- .text(- text: summary(- records, currentSpaceID: currentSpaceID),- annotations: nil, _meta: nil)- ]+ let textContent: Tool.Content = .text(+ text: summary(records, currentSpaceID: currentSpaceID),+ annotations: nil,+ _meta: nil)+ let output = Output(windows: records, currentSpaceId: currentSpaceID)+ if let result = try? CallTool.Result(+ content: [textContent],+ structuredContent: output+ ) {+ return result+ }+ return CallTool.Result(content: [textContent])})diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swiftindex e44e4e6815..d69112f834 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift@@ -1,3 +1,5 @@+import AppKit+import ApplicationServicesimport CoreGraphicsimport CuaDriverCoreimport Foundation@@ -8,12 +10,20 @@ public enum SetValueTool {tool: Tool(name: "set_value",description: """- Directly set an element's AXValue attribute. For controls like sliders,- steppers, and text fields that expose a settable value. Accepts a string- (which the target element will coerce as needed).+ Set a value on a UI element. Two modes depending on element role:- For free-form text entry, prefer `type_text_in` — it inserts at the- cursor rather than replacing the whole value.+ - **AXPopUpButton / select dropdown**: finds the child option whose+ title or value matches `value` (case-insensitive) and AXPresses it+ directly — the native macOS popup menu is never opened, so focus+ is never stolen. Use this for HTML <select> elements in Safari or+ any native NSPopUpButton. Pass the option's display label as `value`+ (e.g. "Blue", not "blue").++ - **All other elements**: writes `AXValue` directly (sliders, steppers,+ date pickers, native text fields that expose settable AXValue).++ For free-form text entry into web inputs, prefer `type_text_chars`+ which synthesises key events — AXValue writes are ignored by WebKit.""",inputSchema: ["type": "object",@@ -68,6 +78,27 @@ public enum SetValueTool {windowId: windowId,elementIndex: index)+ let target = AXInput.describe(element)++ // ── AXPopUpButton (HTML <select>) special path ──────────+ // Safari WebKit does not propagate AXValue writes back to+ // the HTML DOM for <select> elements — the AX write returns+ // success but the JS `element.value` remains unchanged.+ // Instead, iterate the AX children (the option items) and+ // AXPress the one whose AXTitle or AXValue matches `value`.+ // This bypasses the native macOS popup menu entirely, so the+ // option is selected without the menu needing to be visible.+ if target.role == "AXPopUpButton" {+ return try await selectPopupOption(+ element: element,+ index: index,+ pid: pid,+ value: value,+ elementTitle: target.title ?? ""+ )+ }++ // ── Default path: write AXValue directly ────────────────try await AppStateRegistry.focusGuard.withFocusSuppressed(pid: pid,element: element@@ -78,7 +109,6 @@ public enum SetValueTool {value: value as CFTypeRef)}- let target = AXInput.describe(element)let summary ="✅ Set AXValue on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"."return CallTool.Result(@@ -94,6 +124,175 @@ public enum SetValueTool {})+ /// Select a specific option in an AXPopUpButton.+ ///+ /// Strategy (in order):+ /// 1. AX children — works for native AppKit NSPopUpButton where the option+ /// elements are AXMenuItem children of the button even when the popup is closed.+ /// 2. JavaScript injection via `osascript` — used when the target is Safari/WebKit,+ /// whose `<select>` elements expose no AX children without the popup open. Finds+ /// the matching `<option>` by text or value and sets it via the DOM, dispatching+ /// a `change` event. No focus steal — osascript's `do JavaScript` runs against+ /// whichever Safari document is frontmost in the process, regardless of focus.+ private static func selectPopupOption(+ element: AXUIElement,+ index: Int,+ pid: Int32,+ value: String,+ elementTitle: String+ ) async throws -> CallTool.Result {+ // ── Strategy 1: AX children (native AppKit popup buttons) ──────────+ let children = AXInput.children(of: element)+ if !children.isEmpty {+ let valueLower = value.lowercased()+ var matchedIndex = -1+ var availableTitles: [String] = []+ for (i, child) in children.enumerated() {+ let childTitle = AXInput.stringAttribute("AXTitle", of: child) ?? ""+ let childValue = AXInput.stringAttribute("AXValue", of: child) ?? ""+ availableTitles.append(childTitle)+ if childTitle.lowercased() == valueLower+ || childValue.lowercased() == valueLower+ {+ matchedIndex = i+ break+ }+ }+ if matchedIndex >= 0 {+ let option = children[matchedIndex]+ try await AppStateRegistry.focusGuard.withFocusSuppressed(+ pid: pid,+ element: option+ ) {+ try AXInput.performAction("AXPress", on: option)+ }+ let optTitle = AXInput.stringAttribute("AXTitle", of: option) ?? value+ return CallTool.Result(+ content: [.text(+ text: "✅ Selected '\(optTitle)' in AXPopUpButton [\(index)] "+ + "\"\(elementTitle)\" via AX child AXPress.",+ annotations: nil,+ _meta: nil+ )]+ )+ }+ let avail = availableTitles.map { "\"\($0)\"" }.joined(separator: ", ")+ return CallTool.Result(+ content: [.text(+ text: "❌ No AX child matching '\(value)' in AXPopUpButton [\(index)] "+ + "\"\(elementTitle)\". Available: [\(avail)].",+ annotations: nil,+ _meta: nil+ )],+ isError: true+ )+ }++ // ── Strategy 2: JavaScript injection via osascript (Safari/WebKit) ──+ // WebKit-backed <select> elements expose no AX children when the popup+ // is closed. Fall back to running JavaScript in the Safari document.+ let appName = NSRunningApplication(processIdentifier: pid)?.localizedName ?? ""+ guard appName == "Safari" else {+ return CallTool.Result(+ content: [.text(+ text: "❌ AXPopUpButton [\(index)] '\(elementTitle)' has no AX children and "+ + "target is '\(appName)' (not Safari) — no fallback available.",+ annotations: nil,+ _meta: nil+ )],+ isError: true+ )+ }+ return await setSelectViaJS(index: index, elementTitle: elementTitle, value: value)+ }++ /// Set an HTML <select> element's value in Safari via `osascript do JavaScript`.+ /// Searches all <select> elements for an <option> whose text or value matches+ /// `value` (case-insensitive), then sets it and dispatches a `change` event.+ private static func setSelectViaJS(+ index: Int,+ elementTitle: String,+ value: String+ ) async -> CallTool.Result {+ // Percent-encode the (lowercased) value using only unreserved+ // URL characters as the allowed set. This means every special+ // character — including single quotes, double quotes, backslashes,+ // and percent signs — is encoded as %XX, which is safe to embed+ // in both a JS single-quoted string (via decodeURIComponent) and+ // an AppleScript double-quoted string without any additional+ // escaping. No double-escape arithmetic needed.+ var unreserved = CharacterSet.alphanumerics+ unreserved.insert(charactersIn: "-._~")+ let vLow = value.lowercased()+ let vEncoded = vLow.addingPercentEncoding(withAllowedCharacters: unreserved) ?? vLow++ let js =+ "(function(){" ++ "var v=decodeURIComponent('\(vEncoded)');" ++ "var ss=document.querySelectorAll('select'),opts=[];" ++ "for(var i=0;i<ss.length;i++){" ++ "for(var j=0;j<ss[i].options.length;j++){" ++ "var t=ss[i].options[j].text.toLowerCase()," ++ "u=ss[i].options[j].value.toLowerCase();" ++ "opts.push(t+'|'+u);" ++ "if(t===v||u===v){" ++ "ss[i].value=ss[i].options[j].value;" ++ "ss[i].dispatchEvent(new Event('change',{bubbles:true}));" ++ "return 'SET:'+ss[i].value;}}" ++ "}return 'NOTFOUND:'+opts.join(',');" ++ "})()"++ let appleScript = "tell application \"Safari\" to do JavaScript \"\(js)\" in front document"+ let proc = Process()+ proc.executableURL = URL(fileURLWithPath: "/usr/bin/osascript")+ proc.arguments = ["-e", appleScript]+ let outPipe = Pipe()+ proc.standardOutput = outPipe+ proc.standardError = Pipe()+ do {+ try proc.run()+ } catch {+ return errorResult("osascript launch failed: \(error)")+ }+ // Wait for osascript with a 10-second deadline. A stuck Safari+ // permission prompt or unresponsive renderer can cause+ // waitUntilExit() to block indefinitely, which would stall+ // the MCP tool handler permanently. Poll on 50 ms ticks so we+ // stay in the async/Swift concurrency cooperative pool.+ let deadline = Date().addingTimeInterval(10.0)+ while proc.isRunning && Date() < deadline {+ try? await Task.sleep(nanoseconds: 50_000_000)+ }+ if proc.isRunning {+ proc.terminate()+ return errorResult("osascript timed out after 10 seconds")+ }+ let raw = (String(+ data: outPipe.fileHandleForReading.readDataToEndOfFile(),+ encoding: .utf8+ ) ?? "").trimmingCharacters(in: .whitespacesAndNewlines)++ if raw.hasPrefix("SET:") {+ let domVal = String(raw.dropFirst(4))+ return CallTool.Result(+ content: [.text(+ text: "✅ Set select [\(index)] '\(elementTitle)' to '\(value)' via "+ + "Safari JavaScript (DOM value: \"\(domVal)\").",+ annotations: nil,+ _meta: nil+ )]+ )+ }+ if raw.hasPrefix("NOTFOUND:") {+ let available = String(raw.dropFirst(9))+ return errorResult(+ "No <option> matching '\(value)' found in any <select>. "+ + "Available (text|value): \(available)"+ )+ }+ return errorResult("JavaScript returned unexpected output: \(raw.prefix(200))")+ }+private static func isWindowMinimized(pid: Int32) -> Bool {guard let onScreen = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements],diff --git a/libs/cua-driver/Tests/FocusMonitorApp/FocusMonitorApp.swift b/libs/cua-driver/Tests/FocusMonitorApp/FocusMonitorApp.swiftindex f6be5b3e3e..e94d60b69f 100644--- a/libs/cua-driver/Tests/FocusMonitorApp/FocusMonitorApp.swift+++ b/libs/cua-driver/Tests/FocusMonitorApp/FocusMonitorApp.swift@@ -1,19 +1,64 @@/// FocusMonitorApp — counts how many times it loses focus.///-/// On every NSApplication.didResignActiveNotification the counter increments-/// and the current value is written to /tmp/focus_monitor_losses.txt.+/// Tracks three kinds of focus loss:+///+/// 1. **App-level** (NSApplication.didResignActiveNotification): the whole+/// app loses the system focus token. Written to /tmp/focus_monitor_losses.txt.+///+/// 2. **Window/keyboard-level** (NSWindow.didResignKeyNotification): the window+/// loses key status (another window, including a background Safari window,+/// becomes key). Written to /tmp/focus_monitor_key_losses.txt.+///+/// 3. **Text-field first-responder** (TrackingTextField.resignFirstResponder):+/// the editable NSTextField loses first-responder status, meaning another+/// app or window captured keyboard input. Written to+/// /tmp/focus_monitor_field_losses.txt.+///+/// A visible editable NSTextField is created in the window and made first+/// responder immediately, so keyboard input goes to it by default. Agents+/// operating in background-mode should NOT steal this keyboard focus even+/// when typing into other windows.+////// At startup, prints FOCUS_PID=<pid> to stdout so the test harness can/// discover the process.import AppKit+// ---------------------------------------------------------------------------+// Tracking NSTextField — calls back when it loses first-responder status.+// ---------------------------------------------------------------------------++class TrackingTextField: NSTextField {+ var onResignFirstResponder: (() -> Void)?++ override func resignFirstResponder() -> Bool {+ let result = super.resignFirstResponder()+ if result {+ onResignFirstResponder?()+ }+ return result+ }+}++// ---------------------------------------------------------------------------+// App delegate+// ---------------------------------------------------------------------------+class AppDelegate: NSObject, NSApplicationDelegate {var window: NSWindow!- var label: NSTextField!- var lossCount = 0+ var statusLabel: NSTextField!+ var inputField: TrackingTextField!++ var lossCount = 0 // app-level focus losses+ var keyLossCount = 0 // window key-status losses+ var fieldLossCount = 0 // text-field first-responder losses++ let lossFile = "/tmp/focus_monitor_losses.txt"+ let keyFile = "/tmp/focus_monitor_key_losses.txt"+ let fieldFile = "/tmp/focus_monitor_field_losses.txt"func applicationDidFinishLaunching(_ notification: Notification) {- let rect = NSRect(x: 200, y: 200, width: 420, height: 200)+ let rect = NSRect(x: 200, y: 200, width: 460, height: 280)window = NSWindow(contentRect: rect,styleMask: [.titled, .closable, .miniaturizable],@@ -23,42 +68,121 @@ class AppDelegate: NSObject, NSApplicationDelegate {window.title = "Focus Monitor"window.isReleasedWhenClosed = false- label = NSTextField(labelWithString: "focus_losses: 0")- label.font = NSFont.monospacedSystemFont(ofSize: 32, weight: .bold)- label.frame = NSRect(x: 20, y: 60, width: 380, height: 80)- label.alignment = .center- window.contentView?.addSubview(label)+ // ── Status label ─────────────────────────────────────────────────+ statusLabel = NSTextField(labelWithString: "app_losses: 0 | key_losses: 0 | field_losses: 0")+ statusLabel.font = NSFont.monospacedSystemFont(ofSize: 16, weight: .bold)+ statusLabel.frame = NSRect(x: 20, y: 180, width: 420, height: 60)+ statusLabel.alignment = .center+ statusLabel.lineBreakMode = .byWordWrapping+ window.contentView?.addSubview(statusLabel)++ // ── Instruction label ─────────────────────────────────────────────+ let hint = NSTextField(labelWithString: "↓ This input holds keyboard focus ↓")+ hint.font = NSFont.systemFont(ofSize: 12)+ hint.textColor = .secondaryLabelColor+ hint.frame = NSRect(x: 20, y: 148, width: 420, height: 22)+ hint.alignment = .center+ window.contentView?.addSubview(hint)++ // ── Editable text field that acts as the keyboard-focus sentinel ──+ inputField = TrackingTextField()+ inputField.frame = NSRect(x: 60, y: 100, width: 340, height: 36)+ inputField.font = NSFont.systemFont(ofSize: 16)+ inputField.placeholderString = "keyboard focus stays here…"+ inputField.bezelStyle = .roundedBezel+ inputField.onResignFirstResponder = { [weak self] in+ self?.fieldDidResignFirstResponder()+ }+ window.contentView?.addSubview(inputField)++ // ── Secondary label showing the field-focus state ─────────────────+ let fieldHint = NSTextField(labelWithString: "")+ fieldHint.font = NSFont.monospacedSystemFont(ofSize: 12, weight: .regular)+ fieldHint.textColor = .systemOrange+ fieldHint.frame = NSRect(x: 20, y: 60, width: 420, height: 30)+ fieldHint.alignment = .center+ fieldHint.tag = 99+ window.contentView?.addSubview(fieldHint)window.makeKeyAndOrderFront(nil)NSApp.activate(ignoringOtherApps: true)+ // Give the window a run-loop cycle to become key before we+ // make the text field first responder.+ DispatchQueue.main.async {+ self.window.makeFirstResponder(self.inputField)+ }++ // ── Notifications ──────────────────────────────────────────────────NotificationCenter.default.addObserver(self,selector: #selector(appDidResign),name: NSApplication.didResignActiveNotification,object: nil)+ NotificationCenter.default.addObserver(+ self,+ selector: #selector(windowDidResignKey),+ name: NSWindow.didResignKeyNotification,+ object: window+ )+ // Write initial zeros so tests can open the files even before+ // any loss event fires.writeLosses()+ writeKeyLosses()+ writeFieldLosses()+let pid = ProcessInfo.processInfo.processIdentifier- // Flush to stdout so the test harness can read it.print("FOCUS_PID=\(pid)")fflush(stdout)}+ // ── Notification handlers ────────────────────────────────────────────+@objc func appDidResign(_ note: Notification) {lossCount += 1- label.stringValue = "focus_losses: \(lossCount)"+ updateStatusLabel()writeLosses()}+ @objc func windowDidResignKey(_ note: Notification) {+ keyLossCount += 1+ updateStatusLabel()+ writeKeyLosses()+ }++ func fieldDidResignFirstResponder() {+ fieldLossCount += 1+ updateStatusLabel()+ writeFieldLosses()+ // Show which field stole focus via the orange hint label.+ if let lbl = window.contentView?.viewWithTag(99) as? NSTextField {+ lbl.stringValue = "⚠️ keyboard focus left the input field (\(fieldLossCount)x)"+ }+ }++ // ── Helpers ──────────────────────────────────────────────────────────++ func updateStatusLabel() {+ statusLabel.stringValue =+ "app_losses: \(lossCount) | key_losses: \(keyLossCount) | field_losses: \(fieldLossCount)"+ }+func writeLosses() {- let path = "/tmp/focus_monitor_losses.txt"- try? "\(lossCount)".write(toFile: path, atomically: true, encoding: .utf8)+ try? "\(lossCount)".write(toFile: lossFile, atomically: true, encoding: .utf8)+ }++ func writeKeyLosses() {+ try? "\(keyLossCount)".write(toFile: keyFile, atomically: true, encoding: .utf8)+ }++ func writeFieldLosses() {+ try? "\(fieldLossCount)".write(toFile: fieldFile, atomically: true, encoding: .utf8)}}-// Minimal NSApplication bootstrap — no XIB, no storyboard.+// ── Minimal NSApplication bootstrap — no XIB, no storyboard. ──────────────let app = NSApplication.sharedapp.setActivationPolicy(.regular)let delegate = AppDelegate()diff --git a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.htmlnew file mode 100644index 0000000000..44225353a9--- /dev/null+++ b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html@@ -0,0 +1,164 @@+<!DOCTYPE html>+<html>+<head>+<meta charset="utf-8">+<title>All Inputs Test Form</title>+<style>+body { font-family: -apple-system, sans-serif; padding: 30px; background: #f0f0f0; max-width: 640px; }+h1 { font-size: 22px; margin-bottom: 20px; }+.field { margin-bottom: 16px; }+label { display: block; font-weight: bold; margin-bottom: 4px; font-size: 14px; }+input[type=text], input[type=password], input[type=email],+input[type=number], input[type=tel], input[type=url], input[type=search],+textarea, select {+ width: 100%; padding: 8px; font-size: 16px; box-sizing: border-box;+ border: 1px solid #bbb; border-radius: 4px;+}+textarea { height: 80px; resize: vertical; }+.radio-group label, .checkbox-group label { font-weight: normal; display: inline; margin-left: 6px; }+input[type=range] { width: 100%; }+input[type=color] { width: 80px; height: 40px; border: none; cursor: pointer; }+button#submit-btn {+ background: #007AFF; color: white; border: none; padding: 12px 28px;+ font-size: 16px; border-radius: 6px; cursor: pointer; margin-top: 10px;+}+button#submit-btn:hover { background: #0066DD; }+#result {+ margin-top: 24px; padding: 16px; background: white; border: 1px solid #ccc;+ border-radius: 6px; font-size: 13px; white-space: pre-wrap; display: none;+}+#submitted { display: none; font-size: 18px; color: green; font-weight: bold; margin-top: 16px; }+</style>+</head>+<body>+<h1>All Inputs Test Form</h1>+<form id="test-form" onsubmit="handleSubmit(event)">++ <div class="field">+ <label for="f-text">Text</label>+ <input type="text" id="f-text" name="text_field" placeholder="Enter text...">+ </div>++ <div class="field">+ <label for="f-password">Password</label>+ <input type="password" id="f-password" name="password_field" placeholder="Enter password...">+ </div>++ <div class="field">+ <label for="f-email">Email</label>+ <input type="email" id="f-email" name="email_field" placeholder="[email protected]">+ </div>++ <div class="field">+ <label for="f-number">Number</label>+ <input type="number" id="f-number" name="number_field" placeholder="42" min="0" max="999">+ </div>++ <div class="field">+ <label for="f-tel">Phone</label>+ <input type="tel" id="f-tel" name="tel_field" placeholder="555-1234">+ </div>++ <div class="field">+ <label for="f-textarea">Message (Textarea)</label>+ <textarea id="f-textarea" name="textarea_field" placeholder="Type a message..."></textarea>+ </div>++ <div class="field">+ <label for="f-select">Favourite colour (Select)</label>+ <select id="f-select" name="select_field">+ <option value="">-- pick one --</option>+ <option value="red">Red</option>+ <option value="green">Green</option>+ <option value="blue">Blue</option>+ <option value="yellow">Yellow</option>+ </select>+ </div>++ <div class="field checkbox-group">+ <label>Checkbox</label><br>+ <input type="checkbox" id="f-checkbox" name="checkbox_field" value="checked">+ <label for="f-checkbox">I agree to the terms</label>+ </div>++ <div class="field radio-group">+ <label>Radio buttons — Preferred fruit</label><br>+ <input type="radio" id="r-apple" name="radio_field" value="apple">+ <label for="r-apple">Apple</label> + <input type="radio" id="r-banana" name="radio_field" value="banana">+ <label for="r-banana">Banana</label> + <input type="radio" id="r-cherry" name="radio_field" value="cherry">+ <label for="r-cherry">Cherry</label>+ </div>++ <div class="field">+ <label for="f-range">Range / Slider (0–100)</label>+ <input type="range" id="f-range" name="range_field" min="0" max="100" value="50">+ <span id="range-display">50</span>+ </div>++ <div class="field">+ <label for="f-date">Date</label>+ <input type="date" id="f-date" name="date_field">+ </div>++ <div class="field">+ <label for="f-color">Colour picker</label><br>+ <input type="color" id="f-color" name="color_field" value="#4488ff">+ </div>++ <button type="submit" id="submit-btn">Submit Form</button>+</form>++<div id="submitted">✓ Form Submitted!</div>+<div id="result"></div>++<script>+document.getElementById('f-range').addEventListener('input', function() {+ document.getElementById('range-display').textContent = this.value;+});++function handleSubmit(e) {+ e.preventDefault();+ var form = document.getElementById('test-form');+ var data = {+ text: document.getElementById('f-text').value,+ password: document.getElementById('f-password').value,+ email: document.getElementById('f-email').value,+ number: document.getElementById('f-number').value,+ tel: document.getElementById('f-tel').value,+ textarea: document.getElementById('f-textarea').value,+ select: document.getElementById('f-select').value,+ checkbox: document.getElementById('f-checkbox').checked,+ radio: (document.querySelector('input[name=radio_field]:checked') || {}).value || '',+ range: document.getElementById('f-range').value,+ date: document.getElementById('f-date').value,+ color: document.getElementById('f-color').value,+ };+ document.getElementById('submitted').style.display = 'block';+ document.getElementById('result').style.display = 'block';+ document.getElementById('result').textContent = JSON.stringify(data, null, 2);+ window._submitted = data;+}++// Expose live field state for test assertions (no submit required)+function getFieldValues() {+ return JSON.stringify({+ text: document.getElementById('f-text').value,+ password: document.getElementById('f-password').value,+ email: document.getElementById('f-email').value,+ number: document.getElementById('f-number').value,+ tel: document.getElementById('f-tel').value,+ textarea: document.getElementById('f-textarea').value,+ select: document.getElementById('f-select').value,+ checkbox: document.getElementById('f-checkbox').checked,+ radio: (document.querySelector('input[name=radio_field]:checked') || {}).value || '',+ range: document.getElementById('f-range').value,+ date: document.getElementById('f-date').value,+ color: document.getElementById('f-color').value,+ submitted: typeof window._submitted !== 'undefined',+ });+}+</script>+</body>+</html>diff --git a/libs/cua-driver/Tests/integration/test_hermes_form_fill.py b/libs/cua-driver/Tests/integration/test_hermes_form_fill.pynew file mode 100644index 0000000000..bf7b1c873c--- /dev/null+++ b/libs/cua-driver/Tests/integration/test_hermes_form_fill.py@@ -0,0 +1,396 @@+"""Integration test: hermes CLI drives individual form inputs in backgrounded Safari.++Goal+----+One test per HTML input type. Each test asks hermes to interact with exactly+ONE element in a Safari window that is NOT frontmost — verifying that cua-driver+can synthesise events without stealing keyboard/mouse focus from the foreground app+(FocusMonitorApp).++Setup (setUpClass)+------------------+1. Open ``fixtures/form_all_inputs.html`` in Safari with ``open -g`` (background).+2. Launch FocusMonitorApp last so it owns the focus token throughout.++Per-test design+---------------+* Prompt is intentionally minimal: capture Safari, interact with ONE named element.+* We do NOT ask hermes to verify or re-capture — that keeps tool-call count to ≤ 3.+* After hermes returns, the test reads the field value via osascript JS eval and+ asserts the expected value was written.+* The FocusMonitorApp focus-loss counter must not increase.++Run+---+ scripts/test.sh test_hermes_form_fill+ python3 -m pytest Tests/integration/test_hermes_form_fill.py -v+"""++from __future__ import annotations++import json+import os+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import default_binary_path # noqa: E402++# ---------------------------------------------------------------------------+# Paths & constants+# ---------------------------------------------------------------------------++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))++_HTML_FORM = os.path.join(_THIS_DIR, "fixtures", "form_all_inputs.html")+_FORM_URL = f"file://{_HTML_FORM}"++_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"+)+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"+_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt"+_FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt"++_HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes")++# Deferred at module level — not evaluated until setUpClass runs. This+# avoids a KeyError during pytest --collect-only / IDE test discovery /+# lint passes that merely import the module without supplying the key.+# Each test class's setUpClass skips the entire suite when the key is absent.+_ANTHROPIC_KEY: str = os.environ.get("ANTHROPIC_API_KEY", "")++# Use haiku for speed/cost; override with HERMES_TEST_MODEL for a stronger model.+_MODEL = os.environ.get("HERMES_TEST_MODEL", "claude-haiku-4-5-20251001")++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------++def _build_focus_app() -> None:+ if not os.path.exists(_FOCUS_APP_EXE):+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)+++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ return proc, int(line.split("=", 1)[1])+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")+++def _read_file_int(path: str) -> int:+ try:+ with open(path) as f:+ return int(f.read().strip())+ except (FileNotFoundError, ValueError):+ return 0+++def _read_focus_losses() -> int:+ return _read_file_int(_LOSS_FILE)+++def _read_key_losses() -> int:+ return _read_file_int(_KEY_LOSS_FILE)+++def _read_field_losses() -> int:+ return _read_file_int(_FIELD_LOSS_FILE)+++def _js(expr: str) -> str:+ """Evaluate a JS expression in Safari's front document via osascript."""+ script = f'tell application "Safari" to do JavaScript "{expr}" in front document'+ r = subprocess.run(["osascript", "-e", script], capture_output=True, text=True, timeout=10)+ return r.stdout.strip()+++def _wait_for_form(timeout: float = 20.0) -> bool:+ deadline = time.time() + timeout+ while time.time() < deadline:+ if _js("typeof getFieldValues") == "function":+ return True+ time.sleep(0.5)+ return False+++def _field(name: str) -> str:+ raw = _js("getFieldValues()")+ try:+ return str(json.loads(raw).get(name, ""))+ except (json.JSONDecodeError, AttributeError):+ return ""+++def _checkbox_checked() -> bool:+ raw = _js("getFieldValues()")+ try:+ return bool(json.loads(raw).get("checkbox", False))+ except (json.JSONDecodeError, ValueError):+ return False+++# ---------------------------------------------------------------------------+# Base test class+# ---------------------------------------------------------------------------++class _HermesFormBase(unittest.TestCase):+ """Shared setup/teardown and the hermes runner for all per-input tests."""++ _focus_proc: subprocess.Popen+ _focus_pid: int++ @classmethod+ def setUpClass(cls) -> None:+ if not _ANTHROPIC_KEY:+ raise unittest.SkipTest(+ "ANTHROPIC_API_KEY is not set — skipping hermes form-fill tests"+ )++ for f in (_LOSS_FILE, _KEY_LOSS_FILE, _FIELD_LOSS_FILE):+ if os.path.exists(f):+ os.remove(f)++ # Force rebuild FocusMonitorApp when Swift source is newer.+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")+ if (not os.path.exists(_FOCUS_APP_EXE)+ or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)):+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)++ subprocess.run(["pkill", "-x", "Safari"], check=False)+ time.sleep(1.0)+ subprocess.run(["open", "-g", "-a", "Safari", _FORM_URL], check=True)++ if not _wait_for_form(25.0):+ raise RuntimeError(+ f"Safari did not load {_HTML_FORM} within 25 s.\n"+ "Check: (1) file exists, (2) Safari › Develop › Allow JavaScript "+ "from Apple Events is ON."+ )++ # FocusMonitorApp becomes frontmost and keeps focus throughout.+ cls._focus_proc, cls._focus_pid = _launch_focus_app()+ time.sleep(0.5)++ @classmethod+ def tearDownClass(cls) -> None:+ cls._focus_proc.terminate()+ subprocess.run(["pkill", "-x", "Safari"], check=False)++ def setUp(self) -> None:+ self._losses_before = _read_focus_losses()+ self._key_losses_before = _read_key_losses()+ self._field_losses_before = _read_field_losses()++ def _run_hermes(self, prompt: str, timeout: int = 120, max_turns: int = 5) -> str:+ """Run hermes with the anthropic provider and return combined stdout+stderr."""+ env = os.environ.copy()+ env["ANTHROPIC_API_KEY"] = _ANTHROPIC_KEY+ env["HERMES_COMPUTER_USE_BACKEND"] = "cua"+ env["HERMES_CUA_DRIVER_CMD"] = default_binary_path()++ result = subprocess.run(+ [+ _HERMES_BIN, "chat",+ "--provider", "anthropic",+ "-m", _MODEL,+ "--toolsets", "computer_use",+ "--yolo", # auto-approve every computer_use action+ "-Q", # quiet: only final response + session line+ "--max-turns", str(max_turns),+ "-q", prompt,+ ],+ env=env, capture_output=True, text=True, timeout=timeout,+ )+ return (result.stdout + "\n" + result.stderr).strip()++ def _assert_no_new_focus_loss(self) -> None:+ """Assert no new app-level OR key-level focus losses occurred."""+ time.sleep(0.3) # let any async activation settle+ app_delta = _read_focus_losses() - self._losses_before+ key_delta = _read_key_losses() - self._key_losses_before+ field_delta = _read_field_losses() - self._field_losses_before+ msgs = []+ if app_delta:+ msgs.append(f"app lost focus {app_delta}x")+ if key_delta:+ msgs.append(f"window lost key status {key_delta}x")+ if field_delta:+ msgs.append(f"text-input lost first-responder {field_delta}x")+ self.assertEqual(+ len(msgs), 0,+ f"Focus stolen in {self._testMethodName}: "+ + "; ".join(msgs)+ + " — background events must not steal focus",+ )+++# ---------------------------------------------------------------------------+# One test class per input type+# ---------------------------------------------------------------------------++class TestTextInput(_HermesFormBase):+ """<input type=text> — fill with a short string."""++ def test_text_input(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari' to see the form. "+ "Then AX-click the Text input (id='f-text') to focus it, "+ "then type 'Hello World' into it. Stop after typing."+ )+ val = _field("text")+ self.assertEqual(val, "Hello World",+ msg=f"text='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++class TestPasswordInput(_HermesFormBase):+ """<input type=password> — fill with a short string."""++ def test_password_input(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari'. "+ "Then AX-click the Password input (id='f-password') and type 'Pass1234!'. "+ "Stop after typing."+ )+ val = _field("password")+ self.assertEqual(val, "Pass1234!",+ msg=f"password='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++class TestEmailInput(_HermesFormBase):+ """<input type=email> — fill with an email address."""++ def test_email_input(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari'. "+ "Then AX-click the Email input (id='f-email') and type '[email protected]'. "+ "Stop after typing."+ )+ val = _field("email")+ self.assertEqual(val, "[email protected]",+ msg=f"email='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++class TestNumberInput(_HermesFormBase):+ """<input type=number> — fill with a number."""++ def test_number_input(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari'. "+ "Then AX-click the Number input (id='f-number') and type '42'. "+ "Stop after typing."+ )+ val = _field("number")+ self.assertEqual(val, "42",+ msg=f"number='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++class TestTextarea(_HermesFormBase):+ """<textarea> — fill with a sentence."""++ def test_textarea(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari'. "+ "Then AX-click the Message textarea (id='f-textarea') and type 'bg works'. "+ "Stop after typing."+ )+ val = _field("textarea")+ self.assertIn("bg works", val,+ msg=f"textarea='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++class TestCheckbox(_HermesFormBase):+ """<input type=checkbox> — check it."""++ def test_checkbox(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari'. "+ "Then AX-click the checkbox labelled 'I agree to the terms' (id='f-checkbox'). "+ "Stop after clicking."+ )+ self.assertTrue(_checkbox_checked(),+ msg=f"checkbox not checked\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++class TestSelectDropdown(_HermesFormBase):+ """<select> — pick an option from the dropdown.++ Strategy: click the AXPopUpButton (which now returns available options ++ a hint to use set_value), then call set_value with 'Blue'. The set_value+ tool finds the 'Blue' child option and AXPresses it directly without+ opening the native macOS popup menu.+ """++ def test_select_dropdown(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari' to see the AX tree. "+ "Find the element for 'Favourite colour (Select)' (an AXPopUpButton, id='f-select'). "+ "Use computer_use: action='set_value', element=<index>, value='Blue' to select "+ "the Blue option directly without opening the native menu. Stop after setting.",+ max_turns=4,+ timeout=180,+ )+ val = _field("select")+ self.assertEqual(val, "blue",+ msg=f"select='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++class TestRadioButton(_HermesFormBase):+ """<input type=radio> — select the Banana option."""++ def test_radio_button(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari'. "+ "Then AX-click the 'Banana' radio button (id='r-banana'). "+ "Stop after clicking."+ )+ val = _field("radio")+ self.assertEqual(val, "banana",+ msg=f"radio='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++class TestSubmitButton(_HermesFormBase):+ """<button type=submit> — click Submit and confirm the page responds."""++ def test_submit_button(self) -> None:+ out = self._run_hermes(+ "Safari is in the background with a test form open. "+ "Use computer_use: action='capture', app='Safari' to see the AX tree. "+ "Find the AXButton labelled 'Submit Form' (id='submit-btn'). "+ "Click it using its element index. Stop after clicking."+ )+ submitted = _js("(window._submitted !== undefined).toString()")+ self.assertEqual(submitted, "true",+ msg=f"form not submitted\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss()+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver/Tests/integration/test_overlay_z_order.py b/libs/cua-driver/Tests/integration/test_overlay_z_order.pynew file mode 100644index 0000000000..ea48239e92--- /dev/null+++ b/libs/cua-driver/Tests/integration/test_overlay_z_order.py@@ -0,0 +1,213 @@+"""Integration test: agent-cursor overlay z-ordering.++Verifies that after driving a backgrounded window the cua-driver overlay+is z-ordered JUST ABOVE the target window (NSWindowLevel.normal + ordered+above target) rather than at NSWindowLevel.floating (above ALL normal windows).++Expected ordering after pinning above a background window:+ [... , target-window , overlay , windows-that-were-above-target , ...]++Two assertions are verified:++1. **overlay appears at layer 0** — the overlay's CGWindowLayer is 0+ (NSWindowLevel.normal), so it shows up in list_windows. At .floating+ (level 3) it would be absent from the layer-0 filtered list.++2. **sandwich ordering** — overlay z_index > target z_index, and any+ window that was above the target BEFORE the click remains above the+ overlay AFTER the click.++Run:+ CUA_DRIVER_BINARY=.build/CuaDriver.app/Contents/MacOS/cua-driver \\+ python3 -m unittest test_overlay_z_order -v+"""++from __future__ import annotations++import os+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import DriverClient, default_binary_path, resolve_window_id # noqa: E402++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))+_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"+)++CALCULATOR_BUNDLE = "com.apple.calculator"+++def _ensure_focus_app_built() -> None:+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")+ if not os.path.exists(_FOCUS_APP_EXE) or (+ os.path.exists(src) and os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)+ ):+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)+++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ return proc, int(line.split("=", 1)[1])+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")+++class TestOverlayZOrder(unittest.TestCase):+ """Overlay is sandwiched just above the target window, not floating above all."""++ _focus_proc: subprocess.Popen+ _focus_pid: int++ @classmethod+ def setUpClass(cls) -> None:+ _ensure_focus_app_built()+ # Kill Calculator if stale+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ time.sleep(0.3)++ @classmethod+ def tearDownClass(cls) -> None:+ subprocess.run(["pkill", "-x", "Calculator"], check=False)++ def setUp(self) -> None:+ self.client = DriverClient(default_binary_path()).__enter__()+ # Ensure agent cursor is enabled for this test session.+ self.client.call_tool("set_agent_cursor_enabled", {"enabled": True})++ # Launch Calculator in background (it may briefly become frontmost).+ r = self.client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE})+ self.calc_pid = r["structuredContent"]["pid"]+ time.sleep(0.5)++ # Launch FocusMonitorApp as frontmost so Calculator is definitely behind it.+ self._focus_proc, self._focus_pid = _launch_focus_app()+ time.sleep(0.5) # let it settle as the frontmost window++ def tearDown(self) -> None:+ self._focus_proc.terminate()+ subprocess.run(["pkill", "-x", "Calculator"], check=False)+ self.client.__exit__(None, None, None)++ # ------------------------------------------------------------------++ def test_overlay_at_normal_level_above_target(self) -> None:+ """Overlay appears at layer=0 (not .floating/layer=3) and z_index > target."""+ # Snapshot all on-screen windows BEFORE the click.+ all_before = self._all_on_screen_windows()+ driver_pid = self.client.process.pid++ calc_wins_before = [w for w in all_before if w["pid"] == self.calc_pid]+ self.assertTrue(calc_wins_before, "Calculator window not visible before click")+ calc_z_before = max(w["z_index"] for w in calc_wins_before)+ calc_win_before = max(calc_wins_before, key=lambda w: w["z_index"])+ calc_win_id = calc_win_before["window_id"]++ # Use a pixel-addressed click at the center of the Calculator window+ # so we don't need get_window_state element indices.+ b = calc_win_before["bounds"]+ cx = b["width"] / 2.0+ cy = b["height"] / 2.0++ # Windows above Calculator that are NOT the driver overlay.+ fg_wins_before = [+ w for w in all_before+ if w["z_index"] > calc_z_before+ and w["pid"] != driver_pid+ and w.get("is_on_screen")+ ]++ # Trigger a pixel click to make the overlay appear and pin above Calculator.+ self.client.call_tool(+ "click",+ {+ "pid": self.calc_pid,+ "window_id": calc_win_id,+ "x": cx,+ "y": cy,+ },+ )++ # Let the defensive-repin ticks fully settle (last tick fires at ~1200ms+ # after the second pinAbove inside click, which returns after animations).+ time.sleep(1.5)++ # ── Assertion 1: overlay is visible at layer 0 ─────────────────────────+ all_after = self._all_on_screen_windows()+ overlay_wins = [w for w in all_after if w["pid"] == driver_pid]+ self.assertTrue(+ overlay_wins,+ f"cua-driver overlay NOT visible in list_windows at layer=0 "+ f"(driver pid={driver_pid}). "+ f"Expected NSWindowLevel.normal overlay to appear as a layer-0 window. "+ f"If level is still .floating (layer 3) it would be filtered out here.",+ )++ overlay_z = max(w["z_index"] for w in overlay_wins)++ # ── Assertion 2: overlay is above the target ────────────────────────────+ calc_wins_after = [w for w in all_after if w["pid"] == self.calc_pid]+ calc_z_after = (+ max(w["z_index"] for w in calc_wins_after) if calc_wins_after else 0+ )+ self.assertGreater(+ overlay_z,+ calc_z_after,+ f"overlay (z={overlay_z}) must be ABOVE Calculator (z={calc_z_after}). "+ f"pinAbove must order the overlay at z_target+1.",+ )++ # ── Assertion 3: foreground windows remain above the overlay ────────────+ # Any window that was above Calculator BEFORE the click and is still+ # on-screen AFTER the click must have z_index > overlay_z.+ # This is the key invariant that distinguishes .normal from .floating.+ missed_above = []+ for w_before in fg_wins_before:+ w_after = next(+ (a for a in all_after if a["window_id"] == w_before["window_id"]),+ None,+ )+ if w_after and w_after.get("is_on_screen"):+ if w_after["z_index"] <= overlay_z:+ missed_above.append(+ f"{w_before['app_name']} "+ f"(window_id={w_before['window_id']}, "+ f"z_before={w_before['z_index']}, "+ f"z_after={w_after['z_index']}, overlay_z={overlay_z})"+ )++ if fg_wins_before and missed_above:+ self.fail(+ "Window(s) that were ABOVE Calculator before the click are now "+ "BELOW OR EQUAL TO the overlay — overlay is NOT sandwiched:\n"+ + "\n".join(f" {m}" for m in missed_above)+ + "\nExpected ordering: [target, overlay, fg-windows]. "+ "This means the overlay is still at .floating level instead of .normal."+ )++ # If there were no foreground windows above Calculator, just note it.+ if not fg_wins_before:+ # Calculator was already frontmost; z-sandwich can't be verified.+ # Assertion 1 + 2 are sufficient in this case.+ pass++ # ------------------------------------------------------------------++ def _all_on_screen_windows(self) -> list[dict]:+ result = self.client.call_tool("list_windows", {"on_screen_only": True})+ return result["structuredContent"]["windows"]+++if __name__ == "__main__":+ unittest.main(verbosity=2)