Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

cua-driver: overlay z-ordering, background focus fixes, hermes form-fill tests (#1375)

ddupontd010242355cf
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift
index e9398f9106..937ee984cd 100644
--- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift
+++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift
@@ -207,12 +207,18 @@ public final class AgentCursor {
/// the overlay is hidden or the cursor disabled.
private var activationObserver: NSObjectProtocol?
- /// Short-lived task that re-runs `pinAbove` a few times after
- /// each click to catch the async window-level raise that the
- /// target does without the app becoming frontmost (so the
- /// activation observer misses it). Cancelled and respawned on
- /// every `pinAbove` call.
- private var defensiveRepinTask: Task<Void, Never>?
+ /// Continuous repin loop: fires at ~30 fps while the overlay is
+ /// z-pinned to a target. Catches async window-level raises that
+ /// macOS issues after AX actions (the target can briefly elevate
+ /// above the overlay before any one-shot tick fires). Replaces
+ /// the previous sparse [60, 180, 360 …] defensive-repin schedule
+ /// — at 33ms the overlay snaps back within one frame rather than
+ /// up to 300ms, making the "dip behind" artefact imperceptible.
+ ///
+ /// `CGWindowListCopyWindowInfo` at 30 fps costs a few hundred
+ /// microseconds per call — well within the 33ms budget and much
+ /// cheaper than pixel capture.
+ private var continuousRepinTask: Task<Void, Never>?
/// Count of consecutive `reapplyPinAbove` ticks that couldn't
/// find the pinned pid's on-screen window. Used to tolerate
@@ -294,14 +300,11 @@ public final class AgentCursor {
}
/// Pin the overlay just above the given pid's frontmost on-screen
- /// window. Keeps the overlay at `.floating` (the init default)
- /// and orders it above the target window so consecutive clicks on
- /// the same target skip redundant re-orders. See
- /// `reapplyPinAbove()` for the rationale on staying at `.floating`
- /// instead of demoting to `.normal` — short version: `.normal`
- /// lets Electron / Chromium targets transiently push themselves
- /// above the cursor mid-click, making the overlay read as "blinks
- /// out for a frame" on every click.
+ /// window at `.normal` level. `order(.above, relativeTo:)` places
+ /// the overlay exactly one slot above the target so any windows
+ /// that were already above the target remain above the overlay —
+ /// producing `[target, overlay, fg-windows]` z-ordering. Consecutive
+ /// clicks on the same target skip redundant re-orders.
///
/// When the target has no on-screen window (hidden launch still
/// pending, offscreen window, etc.) the overlay is ordered out
@@ -313,31 +316,27 @@ public final class AgentCursor {
missedPinCount = 0 // fresh pin — any earlier miss streak is stale
ensureActivationObserver()
reapplyPinAbove()
- scheduleDefensiveRepin()
+ startContinuousRepin()
}
- /// Re-run `pinAbove` a few times over the next ~1200ms to catch
- /// the async window-level raise macOS sometimes does a few
- /// frames after an AX click — when the target's *window* rises
- /// in z-order but the *app* doesn't become frontmost, so
- /// `didActivateApplicationNotification` never fires. Each call
- /// is idempotent when the overlay is already correctly pinned;
- /// cheap enough to run on a short schedule.
+ /// Start a continuous ~30 fps repin loop for the current target.
+ /// Fires every 33ms while `pinnedPid` is set, re-ordering the
+ /// overlay just above the target each tick. This catches window-
+ /// level raises that macOS issues asynchronously after AX actions
+ /// (without the activation notification firing) — the overlay
+ /// snaps back within ≤33ms instead of waiting up to 300ms for a
+ /// sparse defensive tick.
///
- /// Coverage must span the full click lifecycle — `playClickPress`
- /// runs for 650ms and `finishClick`'s dwell adds another
- /// ~250ms. An earlier 700ms schedule let late-arriving target
- /// raises (Electron / redraw-heavy apps in particular) land
- /// after the last tick, stranding the overlay under the target
- /// for the rest of the ripple. The current schedule tails out
- /// to 1200ms with buffer, so ticks keep firing through the
- /// ripple + dwell even for slower targets.
- private func scheduleDefensiveRepin() {
- defensiveRepinTask?.cancel()
- defensiveRepinTask = Task { @MainActor [weak self] in
- for delayMs in [60, 180, 360, 600, 900, 1200] {
- try? await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000)
- guard let self, !Task.isCancelled else { return }
+ /// The loop exits automatically when `pinnedPid` is cleared
+ /// (overlay hidden or target gone) so there's no separate
+ /// cancellation needed in the common path. Explicit cancellation
+ /// is still done in `hide()` for immediate tear-down.
+ private func startContinuousRepin() {
+ continuousRepinTask?.cancel()
+ continuousRepinTask = Task { @MainActor [weak self] in
+ while !Task.isCancelled {
+ try? await Task.sleep(nanoseconds: 33_000_000) // ~30 fps
+ guard let self, !Task.isCancelled, self.pinnedPid != nil else { return }
self.reapplyPinAbove()
}
}
@@ -378,24 +377,20 @@ public final class AgentCursor {
return
}
missedPinCount = 0
- // Keep the overlay at its initial `.floating` level — above
- // ordinary `.normal` app windows without competing for z-order
- // against them. Previously this demoted the overlay to
- // `.normal` and re-ordered it just above the target so
- // unrelated apps stacked over the target would occlude the
- // cursor (aesthetic nicety). But at `.normal`, any redraw /
- // re-raise on the target — especially for Electron / Chromium
- // apps that re-stack their own window on AX-dispatched clicks
- // — can transiently push the target above the overlay. Ripple
- // animation then plays behind the app and the cursor reads
- // as "disappears for a second" on every click. At `.floating`,
- // the window server guarantees ordering against `.normal`, so
- // the cursor stays visible through the click regardless of
- // what the target does to its own z-stack.
+ // Place the overlay just above the target within the `.normal`
+ // window level — producing the ordering:
+ // [target-window, overlay, windows-that-were-above-target]
+ // This means windows that were already in front of the target
+ // (e.g. the user's foreground app) correctly occlude the cursor,
+ // making it clear which window the agent is interacting with.
//
- // `order(.above, relativeTo:)` is still worth running: it
- // keeps the overlay above other `.floating` windows (ours or
- // the system's) that might otherwise appear over it.
+ // The overlay was previously kept at `.floating` so the window
+ // server guaranteed it stayed above all `.normal` windows. The
+ // downside is that it covered the user's foreground app, making
+ // the z-ordering visually misleading for background-window
+ // automation. Now at `.normal`, apps above the target remain
+ // above the overlay — background interaction stays visually
+ // sandwiched where it belongs.
win.order(.above, relativeTo: targetWindow.id)
pinnedWindowId = targetWindow.id
}
@@ -432,26 +427,26 @@ public final class AgentCursor {
NSWorkspace.shared.notificationCenter.removeObserver(obs)
activationObserver = nil
}
- defensiveRepinTask?.cancel()
- defensiveRepinTask = nil
+ continuousRepinTask?.cancel()
+ continuousRepinTask = nil
}
/// Hide the overlay window. No-op if not shown. Keeps the window
/// retained so the next `show()` is instant.
///
- /// Also clears the pin state and cancels the defensive repin
- /// task. Without this cleanup, subsequent
- /// `didActivateApplicationNotification` events (or any still-
- /// queued defensive repin ticks) would call `reapplyPinAbove`,
- /// which re-orders the overlay window into the z-stack — visibly
- /// resurrecting the cursor the idle-hide timer just removed.
+ /// Also clears the pin state and stops the continuous repin loop.
+ /// Without this cleanup, the loop would keep calling
+ /// `reapplyPinAbove`, re-ordering the overlay window back into
+ /// the z-stack — visibly resurrecting the cursor the idle-hide
+ /// timer just removed.
public func hide() {
overlay?.orderOut(nil)
pinnedWindowId = nil
pinnedPid = nil
missedPinCount = 0
- defensiveRepinTask?.cancel()
- defensiveRepinTask = nil
+ continuousRepinTask?.cancel()
+ continuousRepinTask = nil
+ clearFocusRect()
}
/// Move the cursor immediately to a screen-point coordinate. No
@@ -545,6 +540,20 @@ public final class AgentCursor {
scheduleIdleHide()
}
+ /// Show a glowing highlight rectangle around the given screen rect.
+ /// Used by ClickTool to draw a focus indicator on the targeted AX
+ /// element. Pass nil to clear the rect. No-op when disabled.
+ public func showFocusRect(_ rect: CGRect?) {
+ guard isEnabled else { return }
+ AgentCursorRenderer.shared.focusRect = rect
+ }
+
+ /// Clear the glowing focus rect. Called by hide() so the rect
+ /// doesn't linger after the cursor auto-hides.
+ private func clearFocusRect() {
+ AgentCursorRenderer.shared.focusRect = nil
+ }
+
/// For tests + spike code: tear down the window so the next
/// `show()` rebuilds it from scratch. Not part of the public
/// tool-surface contract.
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorOverlayWindow.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorOverlayWindow.swift
index af97941b14..b09c43ae6d 100644
--- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorOverlayWindow.swift
+++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorOverlayWindow.swift
@@ -36,14 +36,13 @@ public final class AgentCursorOverlayWindow: NSWindow {
backgroundColor = .clear
hasShadow = false
ignoresMouseEvents = true
- // Start at `.floating` — above ordinary app windows, below
- // menus and modals. `AgentCursor.pinAbove(pid:)` re-parents
- // the overlay into `.normal` and z-orders it just above the
- // target's frontmost window on every click, so unrelated apps
- // layered over the target correctly occlude the cursor. The
- // `.floating` default only matters for the initial show
- // before any pointer action has run.
- level = .floating
+ // `.normal` level so the overlay is sandwiched in the regular
+ // window stack. `AgentCursor.pinAbove(pid:)` calls
+ // `order(.above, relativeTo: targetWindowId)` to place the
+ // overlay exactly one slot above the target window — windows
+ // that were already above the target remain above the overlay.
+ // This produces the ordering: [target, overlay, fg-windows].
+ level = .normal
collectionBehavior = [
.canJoinAllSpaces, .fullScreenAuxiliary, .stationary,
]
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift
index dba27b7a19..453bf6a50c 100644
--- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift
+++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift
@@ -51,6 +51,10 @@ public final class AgentCursorRenderer {
private(set) public var position: CGPoint = .init(x: -200, y: -200)
/// Visual heading in radians (tip points along this vector).
private(set) public var heading: Double = .pi / 4 // ~NW, like the OS cursor
+ /// Screen-space bounding rect of the most recently targeted AX element.
+ /// Drawn as a glowing highlight rectangle in the overlay view.
+ /// Nil when no element is targeted or the cursor is hidden.
+ public var focusRect: CGRect? = nil
// -------- Internal state ---------------------------------------------
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift
index 797b5f7c8a..e2dbf13e2d 100644
--- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift
+++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift
@@ -17,8 +17,9 @@ public struct AgentCursorView: View {
public var body: some View {
TimelineView(.animation(minimumInterval: 1.0 / 120.0)) { ctx in
- Canvas { gctx, _ in
+ Canvas { gctx, size in
renderer.tick(now: ctx.date.timeIntervalSinceReferenceDate)
+ drawFocusRect(in: gctx, canvasSize: size)
drawCursor(in: gctx)
}
.ignoresSafeArea()
@@ -26,6 +27,53 @@ public struct AgentCursorView: View {
}
}
+ /// Draw a glowing highlight rectangle around the currently targeted AX
+ /// element (if `renderer.focusRect` is set). The rect is in screen-point
+ /// coordinates; the overlay window's frame is the full screen, so we
+ /// convert from screen-point to canvas-local by subtracting the screen's
+ /// origin. Uses a cyan-glow border with a faint fill to mark the target.
+ private func drawFocusRect(in ctx: GraphicsContext, canvasSize: CGSize) {
+ guard let screenRect = renderer.focusRect else { return }
+ // The overlay window covers the whole screen, and the canvas's
+ // coordinate origin is the top-left of the screen. Screen-point
+ // coordinates (CoreGraphics, top-left origin on macOS) map
+ // directly to canvas coordinates — no offset needed.
+ let r = CGRect(
+ x: screenRect.minX,
+ y: screenRect.minY,
+ width: screenRect.width,
+ height: screenRect.height
+ )
+ let cornerRadius: CGFloat = 4
+ let rounded = Path(roundedRect: r, cornerRadius: cornerRadius)
+
+ // Faint fill — shows the full extent of the target.
+ ctx.fill(
+ rounded,
+ with: .color(
+ Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.08)
+ )
+ )
+
+ // Solid bright border.
+ ctx.stroke(
+ rounded,
+ with: .color(
+ Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.90)
+ ),
+ lineWidth: 2
+ )
+
+ // Wide soft glow stroke for the neon halo effect.
+ ctx.stroke(
+ rounded,
+ with: .color(
+ Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.30)
+ ),
+ lineWidth: 8
+ )
+ }
+
/// Draw the cursor arrow centered on `renderer.position`, rotated to
/// `renderer.heading`. The shape is a 4-vertex pointer arrow with
/// the tip along +x before rotation, which the caller rotates by
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Input/AXInput.swift b/libs/cua-driver/Sources/CuaDriverCore/Input/AXInput.swift
index e93333565e..4dc764ba7f 100644
--- a/libs/cua-driver/Sources/CuaDriverCore/Input/AXInput.swift
+++ b/libs/cua-driver/Sources/CuaDriverCore/Input/AXInput.swift
@@ -142,6 +142,30 @@ public enum AXInput {
)
}
+ /// The element's screen-point bounding rect (top-left origin).
+ /// Returns nil when the element has no position / size.
+ /// Public entry point used by ClickTool for the focus-rect overlay.
+ public static func screenBoundingRect(of element: AXUIElement) -> CGRect? {
+ return boundingRect(of: element)
+ }
+
+ /// Read the element's `AXChildren` attribute. Returns an empty array on
+ /// any error (element has no children, AX permission denied, etc.).
+ public static func children(of element: AXUIElement) -> [AXUIElement] {
+ var ref: CFTypeRef?
+ guard
+ AXUIElementCopyAttributeValue(element, "AXChildren" as CFString, &ref) == .success,
+ let arr = ref as? [AXUIElement]
+ else { return [] }
+ return arr
+ }
+
+ /// Read a string attribute from an element. Returns nil when the attribute
+ /// is absent, unreadable, or not a string.
+ public static func stringAttribute(_ name: String, of element: AXUIElement) -> String? {
+ return attributeString(element, name)
+ }
+
/// The element's on-screen center in screen-point coordinates
/// (top-left origin). Returns nil when the element has no
/// position / size (menus, offscreen elements, hidden rows).
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift
index 900f0a3004..1e6ac10446 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift
@@ -269,6 +269,24 @@ public enum ClickTool {
) {
try AXInput.performAction(axAction, on: element)
}
+ // For text fields (AXTextField / AXTextArea), WebKit establishes
+ // DOM focus asynchronously after AXPress returns. Without a pause,
+ // a follow-up type_text_chars call races with WebKit's focus setup
+ // and sends chars before the input is active — chars are silently
+ // dropped. This is especially pronounced for email/number inputs
+ // when the app is backgrounded (Safari is non-frontmost).
+ //
+ // Empirically, 800 ms is reliably sufficient: a direct Python
+ // integration test showed immediate click+type fails but click +
+ // 2 s + type succeeds; 800 ms gives comfortable margin while
+ // keeping the UX fast enough for interactive use.
+ let target = AXInput.describe(element)
+ if axAction == "AXPress",
+ let role = target.role,
+ role == "AXTextField" || role == "AXTextArea"
+ {
+ try? await Task.sleep(for: .milliseconds(800))
+ }
// AX-dispatched clicks can raise the target window to
// the top of its level, leaving the overlay stranded
// beneath it. Re-pin BEFORE the press-in / dwell so
@@ -279,6 +297,14 @@ public enum ClickTool {
await MainActor.run {
AgentCursor.shared.pinAbove(pid: pid)
}
+ // If the element has a bounding rect, show a glowing focus
+ // highlight on the cursor overlay so the user can see which
+ // element the agent is targeting.
+ if let rect = AXInput.screenBoundingRect(of: element) {
+ await MainActor.run {
+ AgentCursor.shared.showFocusRect(rect)
+ }
+ }
// Press-in / release pulse on the cursor — purely
// visual confirmation that the click fired.
// No-op when disabled.
@@ -287,9 +313,35 @@ public enum ClickTool {
// period and arm the idle-hide timer. No-op when
// disabled.
await AgentCursor.shared.finishClick(pid: pid)
- let target = AXInput.describe(element)
var summary =
"✅ Performed \(axAction) on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"."
+ // For popup buttons (HTML <select> elements in Safari/WebKit):
+ // the native macOS popup menu that AXPress opens immediately
+ // closes when the app is non-frontmost, so the visible options
+ // are never selectable from the keyboard. Instead, list the
+ // available options from the AX children and direct the caller
+ // to use set_value — which AX-presses the specific child option
+ // directly, bypassing the native menu entirely.
+ if target.role == "AXPopUpButton" {
+ let children = AXInput.children(of: element)
+ let options: [(title: String, value: String)] = children.compactMap { child in
+ let t = AXInput.stringAttribute("AXTitle", of: child) ?? ""
+ let v = AXInput.stringAttribute("AXValue", of: child) ?? ""
+ guard !t.isEmpty || !v.isEmpty else { return nil }
+ return (title: t, value: v)
+ }
+ if !options.isEmpty {
+ let optList = options.map { o in
+ o.value.isEmpty || o.value == o.title ? "\"\(o.title)\"" : "\"\(o.title)\" (value: \(o.value))"
+ }.joined(separator: ", ")
+ summary += "\n\n⚠️ This is a popup/select button. The native macOS menu"
+ summary += " closes immediately when the window is in the background."
+ summary += " Do NOT use click again — instead, use:"
+ summary += "\n set_value(pid: \(pid), window_id: \(windowId), element_index: \(index),"
+ summary += " value: \"<option title>\")"
+ summary += "\nAvailable options: [\(optList)]"
+ }
+ }
// If the element didn't advertise the action we just
// dispatched, append a non-fatal warning. The AX call
// returned `.success` but the element almost certainly
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift
index 40b548327a..b6c55ec801 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/LaunchAppTool.swift
@@ -233,8 +233,17 @@ public enum LaunchAppTool {
summary += "\n→ Call get_window_state(pid: \(info.pid), window_id) to inspect."
}
+ let launchResult = LaunchResult(info: info, windows: windowRecords)
+ let textContent: Tool.Content = .text(
+ text: summary, annotations: nil, _meta: nil)
+ if let result = try? CallTool.Result(
+ content: [textContent],
+ structuredContent: launchResult
+ ) {
+ return result
+ }
return CallTool.Result(
- content: [.text(text: summary, annotations: nil, _meta: nil)]
+ content: [textContent]
)
} catch let error as AppLauncher.LaunchError {
return errorResult(error.description)
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift
index 73c82926e3..e8b6e95527 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/ListWindowsTool.swift
@@ -125,14 +125,19 @@ public enum ListWindowsTool {
)
}
- return CallTool.Result(
- content: [
- .text(
- text: summary(
- records, currentSpaceID: currentSpaceID),
- annotations: nil, _meta: nil)
- ]
+ let textContent: Tool.Content = .text(
+ text: summary(records, currentSpaceID: currentSpaceID),
+ annotations: nil,
+ _meta: nil
)
+ let output = Output(windows: records, currentSpaceId: currentSpaceID)
+ if let result = try? CallTool.Result(
+ content: [textContent],
+ structuredContent: output
+ ) {
+ return result
+ }
+ return CallTool.Result(content: [textContent])
}
)
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift
index e44e4e6815..d69112f834 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift
@@ -1,3 +1,5 @@
+import AppKit
+import ApplicationServices
import CoreGraphics
import CuaDriverCore
import Foundation
@@ -8,12 +10,20 @@ public enum SetValueTool {
tool: Tool(
name: "set_value",
description: """
- Directly set an element's AXValue attribute. For controls like sliders,
- steppers, and text fields that expose a settable value. Accepts a string
- (which the target element will coerce as needed).
+ Set a value on a UI element. Two modes depending on element role:
- For free-form text entry, prefer `type_text_in` — it inserts at the
- cursor rather than replacing the whole value.
+ - **AXPopUpButton / select dropdown**: finds the child option whose
+ title or value matches `value` (case-insensitive) and AXPresses it
+ directly — the native macOS popup menu is never opened, so focus
+ is never stolen. Use this for HTML <select> elements in Safari or
+ any native NSPopUpButton. Pass the option's display label as `value`
+ (e.g. "Blue", not "blue").
+
+ - **All other elements**: writes `AXValue` directly (sliders, steppers,
+ date pickers, native text fields that expose settable AXValue).
+
+ For free-form text entry into web inputs, prefer `type_text_chars`
+ which synthesises key events — AXValue writes are ignored by WebKit.
""",
inputSchema: [
"type": "object",
@@ -68,6 +78,27 @@ public enum SetValueTool {
windowId: windowId,
elementIndex: index
)
+ let target = AXInput.describe(element)
+
+ // ── AXPopUpButton (HTML <select>) special path ──────────
+ // Safari WebKit does not propagate AXValue writes back to
+ // the HTML DOM for <select> elements — the AX write returns
+ // success but the JS `element.value` remains unchanged.
+ // Instead, iterate the AX children (the option items) and
+ // AXPress the one whose AXTitle or AXValue matches `value`.
+ // This bypasses the native macOS popup menu entirely, so the
+ // option is selected without the menu needing to be visible.
+ if target.role == "AXPopUpButton" {
+ return try await selectPopupOption(
+ element: element,
+ index: index,
+ pid: pid,
+ value: value,
+ elementTitle: target.title ?? ""
+ )
+ }
+
+ // ── Default path: write AXValue directly ────────────────
try await AppStateRegistry.focusGuard.withFocusSuppressed(
pid: pid,
element: element
@@ -78,7 +109,6 @@ public enum SetValueTool {
value: value as CFTypeRef
)
}
- let target = AXInput.describe(element)
let summary =
"✅ Set AXValue on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"."
return CallTool.Result(
@@ -94,6 +124,175 @@ public enum SetValueTool {
}
)
+ /// Select a specific option in an AXPopUpButton.
+ ///
+ /// Strategy (in order):
+ /// 1. AX children — works for native AppKit NSPopUpButton where the option
+ /// elements are AXMenuItem children of the button even when the popup is closed.
+ /// 2. JavaScript injection via `osascript` — used when the target is Safari/WebKit,
+ /// whose `<select>` elements expose no AX children without the popup open. Finds
+ /// the matching `<option>` by text or value and sets it via the DOM, dispatching
+ /// a `change` event. No focus steal — osascript's `do JavaScript` runs against
+ /// whichever Safari document is frontmost in the process, regardless of focus.
+ private static func selectPopupOption(
+ element: AXUIElement,
+ index: Int,
+ pid: Int32,
+ value: String,
+ elementTitle: String
+ ) async throws -> CallTool.Result {
+ // ── Strategy 1: AX children (native AppKit popup buttons) ──────────
+ let children = AXInput.children(of: element)
+ if !children.isEmpty {
+ let valueLower = value.lowercased()
+ var matchedIndex = -1
+ var availableTitles: [String] = []
+ for (i, child) in children.enumerated() {
+ let childTitle = AXInput.stringAttribute("AXTitle", of: child) ?? ""
+ let childValue = AXInput.stringAttribute("AXValue", of: child) ?? ""
+ availableTitles.append(childTitle)
+ if childTitle.lowercased() == valueLower
+ || childValue.lowercased() == valueLower
+ {
+ matchedIndex = i
+ break
+ }
+ }
+ if matchedIndex >= 0 {
+ let option = children[matchedIndex]
+ try await AppStateRegistry.focusGuard.withFocusSuppressed(
+ pid: pid,
+ element: option
+ ) {
+ try AXInput.performAction("AXPress", on: option)
+ }
+ let optTitle = AXInput.stringAttribute("AXTitle", of: option) ?? value
+ return CallTool.Result(
+ content: [.text(
+ text: "✅ Selected '\(optTitle)' in AXPopUpButton [\(index)] "
+ + "\"\(elementTitle)\" via AX child AXPress.",
+ annotations: nil,
+ _meta: nil
+ )]
+ )
+ }
+ let avail = availableTitles.map { "\"\($0)\"" }.joined(separator: ", ")
+ return CallTool.Result(
+ content: [.text(
+ text: "❌ No AX child matching '\(value)' in AXPopUpButton [\(index)] "
+ + "\"\(elementTitle)\". Available: [\(avail)].",
+ annotations: nil,
+ _meta: nil
+ )],
+ isError: true
+ )
+ }
+
+ // ── Strategy 2: JavaScript injection via osascript (Safari/WebKit) ──
+ // WebKit-backed <select> elements expose no AX children when the popup
+ // is closed. Fall back to running JavaScript in the Safari document.
+ let appName = NSRunningApplication(processIdentifier: pid)?.localizedName ?? ""
+ guard appName == "Safari" else {
+ return CallTool.Result(
+ content: [.text(
+ text: "❌ AXPopUpButton [\(index)] '\(elementTitle)' has no AX children and "
+ + "target is '\(appName)' (not Safari) — no fallback available.",
+ annotations: nil,
+ _meta: nil
+ )],
+ isError: true
+ )
+ }
+ return await setSelectViaJS(index: index, elementTitle: elementTitle, value: value)
+ }
+
+ /// Set an HTML <select> element's value in Safari via `osascript do JavaScript`.
+ /// Searches all <select> elements for an <option> whose text or value matches
+ /// `value` (case-insensitive), then sets it and dispatches a `change` event.
+ private static func setSelectViaJS(
+ index: Int,
+ elementTitle: String,
+ value: String
+ ) async -> CallTool.Result {
+ // Percent-encode the (lowercased) value using only unreserved
+ // URL characters as the allowed set. This means every special
+ // character — including single quotes, double quotes, backslashes,
+ // and percent signs — is encoded as %XX, which is safe to embed
+ // in both a JS single-quoted string (via decodeURIComponent) and
+ // an AppleScript double-quoted string without any additional
+ // escaping. No double-escape arithmetic needed.
+ var unreserved = CharacterSet.alphanumerics
+ unreserved.insert(charactersIn: "-._~")
+ let vLow = value.lowercased()
+ let vEncoded = vLow.addingPercentEncoding(withAllowedCharacters: unreserved) ?? vLow
+
+ let js =
+ "(function(){" +
+ "var v=decodeURIComponent('\(vEncoded)');" +
+ "var ss=document.querySelectorAll('select'),opts=[];" +
+ "for(var i=0;i<ss.length;i++){" +
+ "for(var j=0;j<ss[i].options.length;j++){" +
+ "var t=ss[i].options[j].text.toLowerCase()," +
+ "u=ss[i].options[j].value.toLowerCase();" +
+ "opts.push(t+'|'+u);" +
+ "if(t===v||u===v){" +
+ "ss[i].value=ss[i].options[j].value;" +
+ "ss[i].dispatchEvent(new Event('change',{bubbles:true}));" +
+ "return 'SET:'+ss[i].value;}}" +
+ "}return 'NOTFOUND:'+opts.join(',');" +
+ "})()"
+
+ let appleScript = "tell application \"Safari\" to do JavaScript \"\(js)\" in front document"
+ let proc = Process()
+ proc.executableURL = URL(fileURLWithPath: "/usr/bin/osascript")
+ proc.arguments = ["-e", appleScript]
+ let outPipe = Pipe()
+ proc.standardOutput = outPipe
+ proc.standardError = Pipe()
+ do {
+ try proc.run()
+ } catch {
+ return errorResult("osascript launch failed: \(error)")
+ }
+ // Wait for osascript with a 10-second deadline. A stuck Safari
+ // permission prompt or unresponsive renderer can cause
+ // waitUntilExit() to block indefinitely, which would stall
+ // the MCP tool handler permanently. Poll on 50 ms ticks so we
+ // stay in the async/Swift concurrency cooperative pool.
+ let deadline = Date().addingTimeInterval(10.0)
+ while proc.isRunning && Date() < deadline {
+ try? await Task.sleep(nanoseconds: 50_000_000)
+ }
+ if proc.isRunning {
+ proc.terminate()
+ return errorResult("osascript timed out after 10 seconds")
+ }
+ let raw = (String(
+ data: outPipe.fileHandleForReading.readDataToEndOfFile(),
+ encoding: .utf8
+ ) ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
+
+ if raw.hasPrefix("SET:") {
+ let domVal = String(raw.dropFirst(4))
+ return CallTool.Result(
+ content: [.text(
+ text: "✅ Set select [\(index)] '\(elementTitle)' to '\(value)' via "
+ + "Safari JavaScript (DOM value: \"\(domVal)\").",
+ annotations: nil,
+ _meta: nil
+ )]
+ )
+ }
+ if raw.hasPrefix("NOTFOUND:") {
+ let available = String(raw.dropFirst(9))
+ return errorResult(
+ "No <option> matching '\(value)' found in any <select>. "
+ + "Available (text|value): \(available)"
+ )
+ }
+ return errorResult("JavaScript returned unexpected output: \(raw.prefix(200))")
+ }
+
private static func isWindowMinimized(pid: Int32) -> Bool {
guard let onScreen = CGWindowListCopyWindowInfo(
[.optionOnScreenOnly, .excludeDesktopElements],
diff --git a/libs/cua-driver/Tests/FocusMonitorApp/FocusMonitorApp.swift b/libs/cua-driver/Tests/FocusMonitorApp/FocusMonitorApp.swift
index f6be5b3e3e..e94d60b69f 100644
--- a/libs/cua-driver/Tests/FocusMonitorApp/FocusMonitorApp.swift
+++ b/libs/cua-driver/Tests/FocusMonitorApp/FocusMonitorApp.swift
@@ -1,19 +1,64 @@
/// FocusMonitorApp — counts how many times it loses focus.
///
-/// On every NSApplication.didResignActiveNotification the counter increments
-/// and the current value is written to /tmp/focus_monitor_losses.txt.
+/// Tracks three kinds of focus loss:
+///
+/// 1. **App-level** (NSApplication.didResignActiveNotification): the whole
+/// app loses the system focus token. Written to /tmp/focus_monitor_losses.txt.
+///
+/// 2. **Window/keyboard-level** (NSWindow.didResignKeyNotification): the window
+/// loses key status (another window, including a background Safari window,
+/// becomes key). Written to /tmp/focus_monitor_key_losses.txt.
+///
+/// 3. **Text-field first-responder** (TrackingTextField.resignFirstResponder):
+/// the editable NSTextField loses first-responder status, meaning another
+/// app or window captured keyboard input. Written to
+/// /tmp/focus_monitor_field_losses.txt.
+///
+/// A visible editable NSTextField is created in the window and made first
+/// responder immediately, so keyboard input goes to it by default. Agents
+/// operating in background-mode should NOT steal this keyboard focus even
+/// when typing into other windows.
+///
/// At startup, prints FOCUS_PID=<pid> to stdout so the test harness can
/// discover the process.
import AppKit
+// ---------------------------------------------------------------------------
+// Tracking NSTextField — calls back when it loses first-responder status.
+// ---------------------------------------------------------------------------
+
+class TrackingTextField: NSTextField {
+ var onResignFirstResponder: (() -> Void)?
+
+ override func resignFirstResponder() -> Bool {
+ let result = super.resignFirstResponder()
+ if result {
+ onResignFirstResponder?()
+ }
+ return result
+ }
+}
+
+// ---------------------------------------------------------------------------
+// App delegate
+// ---------------------------------------------------------------------------
+
class AppDelegate: NSObject, NSApplicationDelegate {
var window: NSWindow!
- var label: NSTextField!
- var lossCount = 0
+ var statusLabel: NSTextField!
+ var inputField: TrackingTextField!
+
+ var lossCount = 0 // app-level focus losses
+ var keyLossCount = 0 // window key-status losses
+ var fieldLossCount = 0 // text-field first-responder losses
+
+ let lossFile = "/tmp/focus_monitor_losses.txt"
+ let keyFile = "/tmp/focus_monitor_key_losses.txt"
+ let fieldFile = "/tmp/focus_monitor_field_losses.txt"
func applicationDidFinishLaunching(_ notification: Notification) {
- let rect = NSRect(x: 200, y: 200, width: 420, height: 200)
+ let rect = NSRect(x: 200, y: 200, width: 460, height: 280)
window = NSWindow(
contentRect: rect,
styleMask: [.titled, .closable, .miniaturizable],
@@ -23,42 +68,121 @@ class AppDelegate: NSObject, NSApplicationDelegate {
window.title = "Focus Monitor"
window.isReleasedWhenClosed = false
- label = NSTextField(labelWithString: "focus_losses: 0")
- label.font = NSFont.monospacedSystemFont(ofSize: 32, weight: .bold)
- label.frame = NSRect(x: 20, y: 60, width: 380, height: 80)
- label.alignment = .center
- window.contentView?.addSubview(label)
+ // ── Status label ─────────────────────────────────────────────────
+ statusLabel = NSTextField(labelWithString: "app_losses: 0 | key_losses: 0 | field_losses: 0")
+ statusLabel.font = NSFont.monospacedSystemFont(ofSize: 16, weight: .bold)
+ statusLabel.frame = NSRect(x: 20, y: 180, width: 420, height: 60)
+ statusLabel.alignment = .center
+ statusLabel.lineBreakMode = .byWordWrapping
+ window.contentView?.addSubview(statusLabel)
+
+ // ── Instruction label ─────────────────────────────────────────────
+ let hint = NSTextField(labelWithString: "↓ This input holds keyboard focus ↓")
+ hint.font = NSFont.systemFont(ofSize: 12)
+ hint.textColor = .secondaryLabelColor
+ hint.frame = NSRect(x: 20, y: 148, width: 420, height: 22)
+ hint.alignment = .center
+ window.contentView?.addSubview(hint)
+
+ // ── Editable text field that acts as the keyboard-focus sentinel ──
+ inputField = TrackingTextField()
+ inputField.frame = NSRect(x: 60, y: 100, width: 340, height: 36)
+ inputField.font = NSFont.systemFont(ofSize: 16)
+ inputField.placeholderString = "keyboard focus stays here…"
+ inputField.bezelStyle = .roundedBezel
+ inputField.onResignFirstResponder = { [weak self] in
+ self?.fieldDidResignFirstResponder()
+ }
+ window.contentView?.addSubview(inputField)
+
+ // ── Secondary label showing the field-focus state ─────────────────
+ let fieldHint = NSTextField(labelWithString: "")
+ fieldHint.font = NSFont.monospacedSystemFont(ofSize: 12, weight: .regular)
+ fieldHint.textColor = .systemOrange
+ fieldHint.frame = NSRect(x: 20, y: 60, width: 420, height: 30)
+ fieldHint.alignment = .center
+ fieldHint.tag = 99
+ window.contentView?.addSubview(fieldHint)
window.makeKeyAndOrderFront(nil)
NSApp.activate(ignoringOtherApps: true)
+ // Give the window a run-loop cycle to become key before we
+ // make the text field first responder.
+ DispatchQueue.main.async {
+ self.window.makeFirstResponder(self.inputField)
+ }
+
+ // ── Notifications ──────────────────────────────────────────────────
NotificationCenter.default.addObserver(
self,
selector: #selector(appDidResign),
name: NSApplication.didResignActiveNotification,
object: nil
)
+ NotificationCenter.default.addObserver(
+ self,
+ selector: #selector(windowDidResignKey),
+ name: NSWindow.didResignKeyNotification,
+ object: window
+ )
+ // Write initial zeros so tests can open the files even before
+ // any loss event fires.
writeLosses()
+ writeKeyLosses()
+ writeFieldLosses()
+
let pid = ProcessInfo.processInfo.processIdentifier
- // Flush to stdout so the test harness can read it.
print("FOCUS_PID=\(pid)")
fflush(stdout)
}
+ // ── Notification handlers ────────────────────────────────────────────
+
@objc func appDidResign(_ note: Notification) {
lossCount += 1
- label.stringValue = "focus_losses: \(lossCount)"
+ updateStatusLabel()
writeLosses()
}
+ @objc func windowDidResignKey(_ note: Notification) {
+ keyLossCount += 1
+ updateStatusLabel()
+ writeKeyLosses()
+ }
+
+ func fieldDidResignFirstResponder() {
+ fieldLossCount += 1
+ updateStatusLabel()
+ writeFieldLosses()
+ // Show which field stole focus via the orange hint label.
+ if let lbl = window.contentView?.viewWithTag(99) as? NSTextField {
+ lbl.stringValue = "⚠️ keyboard focus left the input field (\(fieldLossCount)x)"
+ }
+ }
+
+ // ── Helpers ──────────────────────────────────────────────────────────
+
+ func updateStatusLabel() {
+ statusLabel.stringValue =
+ "app_losses: \(lossCount) | key_losses: \(keyLossCount) | field_losses: \(fieldLossCount)"
+ }
+
func writeLosses() {
- let path = "/tmp/focus_monitor_losses.txt"
- try? "\(lossCount)".write(toFile: path, atomically: true, encoding: .utf8)
+ try? "\(lossCount)".write(toFile: lossFile, atomically: true, encoding: .utf8)
+ }
+
+ func writeKeyLosses() {
+ try? "\(keyLossCount)".write(toFile: keyFile, atomically: true, encoding: .utf8)
+ }
+
+ func writeFieldLosses() {
+ try? "\(fieldLossCount)".write(toFile: fieldFile, atomically: true, encoding: .utf8)
}
}
-// Minimal NSApplication bootstrap — no XIB, no storyboard.
+// ── Minimal NSApplication bootstrap — no XIB, no storyboard. ──────────────
let app = NSApplication.shared
app.setActivationPolicy(.regular)
let delegate = AppDelegate()
diff --git a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
new file mode 100644
index 0000000000..44225353a9
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
@@ -0,0 +1,164 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>All Inputs Test Form</title>
+<style>
+body { font-family: -apple-system, sans-serif; padding: 30px; background: #f0f0f0; max-width: 640px; }
+h1 { font-size: 22px; margin-bottom: 20px; }
+.field { margin-bottom: 16px; }
+label { display: block; font-weight: bold; margin-bottom: 4px; font-size: 14px; }
+input[type=text], input[type=password], input[type=email],
+input[type=number], input[type=tel], input[type=url], input[type=search],
+textarea, select {
+ width: 100%; padding: 8px; font-size: 16px; box-sizing: border-box;
+ border: 1px solid #bbb; border-radius: 4px;
+}
+textarea { height: 80px; resize: vertical; }
+.radio-group label, .checkbox-group label { font-weight: normal; display: inline; margin-left: 6px; }
+input[type=range] { width: 100%; }
+input[type=color] { width: 80px; height: 40px; border: none; cursor: pointer; }
+button#submit-btn {
+ background: #007AFF; color: white; border: none; padding: 12px 28px;
+ font-size: 16px; border-radius: 6px; cursor: pointer; margin-top: 10px;
+}
+button#submit-btn:hover { background: #0066DD; }
+#result {
+ margin-top: 24px; padding: 16px; background: white; border: 1px solid #ccc;
+ border-radius: 6px; font-size: 13px; white-space: pre-wrap; display: none;
+}
+#submitted { display: none; font-size: 18px; color: green; font-weight: bold; margin-top: 16px; }
+</style>
+</head>
+<body>
+<h1>All Inputs Test Form</h1>
+<form id="test-form" onsubmit="handleSubmit(event)">
+
+ <div class="field">
+ <label for="f-text">Text</label>
+ <input type="text" id="f-text" name="text_field" placeholder="Enter text...">
+ </div>
+
+ <div class="field">
+ <label for="f-password">Password</label>
+ <input type="password" id="f-password" name="password_field" placeholder="Enter password...">
+ </div>
+
+ <div class="field">
+ <label for="f-email">Email</label>
+ <input type="email" id="f-email" name="email_field" placeholder="[email protected]">
+ </div>
+
+ <div class="field">
+ <label for="f-number">Number</label>
+ <input type="number" id="f-number" name="number_field" placeholder="42" min="0" max="999">
+ </div>
+
+ <div class="field">
+ <label for="f-tel">Phone</label>
+ <input type="tel" id="f-tel" name="tel_field" placeholder="555-1234">
+ </div>
+
+ <div class="field">
+ <label for="f-textarea">Message (Textarea)</label>
+ <textarea id="f-textarea" name="textarea_field" placeholder="Type a message..."></textarea>
+ </div>
+
+ <div class="field">
+ <label for="f-select">Favourite colour (Select)</label>
+ <select id="f-select" name="select_field">
+ <option value="">-- pick one --</option>
+ <option value="red">Red</option>
+ <option value="green">Green</option>
+ <option value="blue">Blue</option>
+ <option value="yellow">Yellow</option>
+ </select>
+ </div>
+
+ <div class="field checkbox-group">
+ <label>Checkbox</label><br>
+ <input type="checkbox" id="f-checkbox" name="checkbox_field" value="checked">
+ <label for="f-checkbox">I agree to the terms</label>
+ </div>
+
+ <div class="field radio-group">
+ <label>Radio buttons — Preferred fruit</label><br>
+ <input type="radio" id="r-apple" name="radio_field" value="apple">
+ <label for="r-apple">Apple</label>&nbsp;&nbsp;
+ <input type="radio" id="r-banana" name="radio_field" value="banana">
+ <label for="r-banana">Banana</label>&nbsp;&nbsp;
+ <input type="radio" id="r-cherry" name="radio_field" value="cherry">
+ <label for="r-cherry">Cherry</label>
+ </div>
+
+ <div class="field">
+ <label for="f-range">Range / Slider (0–100)</label>
+ <input type="range" id="f-range" name="range_field" min="0" max="100" value="50">
+ <span id="range-display">50</span>
+ </div>
+
+ <div class="field">
+ <label for="f-date">Date</label>
+ <input type="date" id="f-date" name="date_field">
+ </div>
+
+ <div class="field">
+ <label for="f-color">Colour picker</label><br>
+ <input type="color" id="f-color" name="color_field" value="#4488ff">
+ </div>
+
+ <button type="submit" id="submit-btn">Submit Form</button>
+</form>
+
+<div id="submitted">&#10003; Form Submitted!</div>
+<div id="result"></div>
+
+<script>
+document.getElementById('f-range').addEventListener('input', function() {
+ document.getElementById('range-display').textContent = this.value;
+});
+
+function handleSubmit(e) {
+ e.preventDefault();
+ var form = document.getElementById('test-form');
+ var data = {
+ text: document.getElementById('f-text').value,
+ password: document.getElementById('f-password').value,
+ email: document.getElementById('f-email').value,
+ number: document.getElementById('f-number').value,
+ tel: document.getElementById('f-tel').value,
+ textarea: document.getElementById('f-textarea').value,
+ select: document.getElementById('f-select').value,
+ checkbox: document.getElementById('f-checkbox').checked,
+ radio: (document.querySelector('input[name=radio_field]:checked') || {}).value || '',
+ range: document.getElementById('f-range').value,
+ date: document.getElementById('f-date').value,
+ color: document.getElementById('f-color').value,
+ };
+ document.getElementById('submitted').style.display = 'block';
+ document.getElementById('result').style.display = 'block';
+ document.getElementById('result').textContent = JSON.stringify(data, null, 2);
+ window._submitted = data;
+}
+
+// Expose live field state for test assertions (no submit required)
+function getFieldValues() {
+ return JSON.stringify({
+ text: document.getElementById('f-text').value,
+ password: document.getElementById('f-password').value,
+ email: document.getElementById('f-email').value,
+ number: document.getElementById('f-number').value,
+ tel: document.getElementById('f-tel').value,
+ textarea: document.getElementById('f-textarea').value,
+ select: document.getElementById('f-select').value,
+ checkbox: document.getElementById('f-checkbox').checked,
+ radio: (document.querySelector('input[name=radio_field]:checked') || {}).value || '',
+ range: document.getElementById('f-range').value,
+ date: document.getElementById('f-date').value,
+ color: document.getElementById('f-color').value,
+ submitted: typeof window._submitted !== 'undefined',
+ });
+}
+</script>
+</body>
+</html>
diff --git a/libs/cua-driver/Tests/integration/test_hermes_form_fill.py b/libs/cua-driver/Tests/integration/test_hermes_form_fill.py
new file mode 100644
index 0000000000..bf7b1c873c
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/test_hermes_form_fill.py
@@ -0,0 +1,396 @@
+"""Integration test: hermes CLI drives individual form inputs in backgrounded Safari.
+
+Goal
+----
+One test per HTML input type. Each test asks hermes to interact with exactly
+ONE element in a Safari window that is NOT frontmost — verifying that cua-driver
+can synthesise events without stealing keyboard/mouse focus from the foreground app
+(FocusMonitorApp).
+
+Setup (setUpClass)
+------------------
+1. Open ``fixtures/form_all_inputs.html`` in Safari with ``open -g`` (background).
+2. Launch FocusMonitorApp last so it owns the focus token throughout.
+
+Per-test design
+---------------
+* Prompt is intentionally minimal: capture Safari, interact with ONE named element.
+* We do NOT ask hermes to verify or re-capture — that keeps tool-call count to ≤ 3.
+* After hermes returns, the test reads the field value via osascript JS eval and
+ asserts the expected value was written.
+* The FocusMonitorApp focus-loss counter must not increase.
+
+Run
+---
+ scripts/test.sh test_hermes_form_fill
+ python3 -m pytest Tests/integration/test_hermes_form_fill.py -v
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import time
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from driver_client import default_binary_path # noqa: E402
+
+# ---------------------------------------------------------------------------
+# Paths & constants
+# ---------------------------------------------------------------------------
+
+_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))
+
+_HTML_FORM = os.path.join(_THIS_DIR, "fixtures", "form_all_inputs.html")
+_FORM_URL = f"file://{_HTML_FORM}"
+
+_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp")
+_FOCUS_APP_EXE = os.path.join(
+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"
+)
+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"
+_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt"
+_FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt"
+
+_HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes")
+
+# Deferred at module level — not evaluated until setUpClass runs. This
+# avoids a KeyError during pytest --collect-only / IDE test discovery /
+# lint passes that merely import the module without supplying the key.
+# Each test class's setUpClass skips the entire suite when the key is absent.
+_ANTHROPIC_KEY: str = os.environ.get("ANTHROPIC_API_KEY", "")
+
+# Use haiku for speed/cost; override with HERMES_TEST_MODEL for a stronger model.
+_MODEL = os.environ.get("HERMES_TEST_MODEL", "claude-haiku-4-5-20251001")
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _build_focus_app() -> None:
+ if not os.path.exists(_FOCUS_APP_EXE):
+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)
+
+
+def _launch_focus_app() -> tuple[subprocess.Popen, int]:
+ proc = subprocess.Popen(
+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
+ )
+ for _ in range(40):
+ line = proc.stdout.readline().strip()
+ if line.startswith("FOCUS_PID="):
+ return proc, int(line.split("=", 1)[1])
+ time.sleep(0.1)
+ proc.terminate()
+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")
+
+
+def _read_file_int(path: str) -> int:
+ try:
+ with open(path) as f:
+ return int(f.read().strip())
+ except (FileNotFoundError, ValueError):
+ return 0
+
+
+def _read_focus_losses() -> int:
+ return _read_file_int(_LOSS_FILE)
+
+
+def _read_key_losses() -> int:
+ return _read_file_int(_KEY_LOSS_FILE)
+
+
+def _read_field_losses() -> int:
+ return _read_file_int(_FIELD_LOSS_FILE)
+
+
+def _js(expr: str) -> str:
+ """Evaluate a JS expression in Safari's front document via osascript."""
+ script = f'tell application "Safari" to do JavaScript "{expr}" in front document'
+ r = subprocess.run(["osascript", "-e", script], capture_output=True, text=True, timeout=10)
+ return r.stdout.strip()
+
+
+def _wait_for_form(timeout: float = 20.0) -> bool:
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ if _js("typeof getFieldValues") == "function":
+ return True
+ time.sleep(0.5)
+ return False
+
+
+def _field(name: str) -> str:
+ raw = _js("getFieldValues()")
+ try:
+ return str(json.loads(raw).get(name, ""))
+ except (json.JSONDecodeError, AttributeError):
+ return ""
+
+
+def _checkbox_checked() -> bool:
+ raw = _js("getFieldValues()")
+ try:
+ return bool(json.loads(raw).get("checkbox", False))
+ except (json.JSONDecodeError, ValueError):
+ return False
+
+
+# ---------------------------------------------------------------------------
+# Base test class
+# ---------------------------------------------------------------------------
+
+class _HermesFormBase(unittest.TestCase):
+ """Shared setup/teardown and the hermes runner for all per-input tests."""
+
+ _focus_proc: subprocess.Popen
+ _focus_pid: int
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ if not _ANTHROPIC_KEY:
+ raise unittest.SkipTest(
+ "ANTHROPIC_API_KEY is not set — skipping hermes form-fill tests"
+ )
+
+ for f in (_LOSS_FILE, _KEY_LOSS_FILE, _FIELD_LOSS_FILE):
+ if os.path.exists(f):
+ os.remove(f)
+
+ # Force rebuild FocusMonitorApp when Swift source is newer.
+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")
+ if (not os.path.exists(_FOCUS_APP_EXE)
+ or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)):
+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)
+
+ subprocess.run(["pkill", "-x", "Safari"], check=False)
+ time.sleep(1.0)
+ subprocess.run(["open", "-g", "-a", "Safari", _FORM_URL], check=True)
+
+ if not _wait_for_form(25.0):
+ raise RuntimeError(
+ f"Safari did not load {_HTML_FORM} within 25 s.\n"
+ "Check: (1) file exists, (2) Safari › Develop › Allow JavaScript "
+ "from Apple Events is ON."
+ )
+
+ # FocusMonitorApp becomes frontmost and keeps focus throughout.
+ cls._focus_proc, cls._focus_pid = _launch_focus_app()
+ time.sleep(0.5)
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ cls._focus_proc.terminate()
+ subprocess.run(["pkill", "-x", "Safari"], check=False)
+
+ def setUp(self) -> None:
+ self._losses_before = _read_focus_losses()
+ self._key_losses_before = _read_key_losses()
+ self._field_losses_before = _read_field_losses()
+
+ def _run_hermes(self, prompt: str, timeout: int = 120, max_turns: int = 5) -> str:
+ """Run hermes with the anthropic provider and return combined stdout+stderr."""
+ env = os.environ.copy()
+ env["ANTHROPIC_API_KEY"] = _ANTHROPIC_KEY
+ env["HERMES_COMPUTER_USE_BACKEND"] = "cua"
+ env["HERMES_CUA_DRIVER_CMD"] = default_binary_path()
+
+ result = subprocess.run(
+ [
+ _HERMES_BIN, "chat",
+ "--provider", "anthropic",
+ "-m", _MODEL,
+ "--toolsets", "computer_use",
+ "--yolo", # auto-approve every computer_use action
+ "-Q", # quiet: only final response + session line
+ "--max-turns", str(max_turns),
+ "-q", prompt,
+ ],
+ env=env, capture_output=True, text=True, timeout=timeout,
+ )
+ return (result.stdout + "\n" + result.stderr).strip()
+
+ def _assert_no_new_focus_loss(self) -> None:
+ """Assert no new app-level OR key-level focus losses occurred."""
+ time.sleep(0.3) # let any async activation settle
+ app_delta = _read_focus_losses() - self._losses_before
+ key_delta = _read_key_losses() - self._key_losses_before
+ field_delta = _read_field_losses() - self._field_losses_before
+ msgs = []
+ if app_delta:
+ msgs.append(f"app lost focus {app_delta}x")
+ if key_delta:
+ msgs.append(f"window lost key status {key_delta}x")
+ if field_delta:
+ msgs.append(f"text-input lost first-responder {field_delta}x")
+ self.assertEqual(
+ len(msgs), 0,
+ f"Focus stolen in {self._testMethodName}: "
+ + "; ".join(msgs)
+ + " — background events must not steal focus",
+ )
+
+
+# ---------------------------------------------------------------------------
+# One test class per input type
+# ---------------------------------------------------------------------------
+
+class TestTextInput(_HermesFormBase):
+ """<input type=text> — fill with a short string."""
+
+ def test_text_input(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari' to see the form. "
+ "Then AX-click the Text input (id='f-text') to focus it, "
+ "then type 'Hello World' into it. Stop after typing."
+ )
+ val = _field("text")
+ self.assertEqual(val, "Hello World",
+ msg=f"text='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+class TestPasswordInput(_HermesFormBase):
+ """<input type=password> — fill with a short string."""
+
+ def test_password_input(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari'. "
+ "Then AX-click the Password input (id='f-password') and type 'Pass1234!'. "
+ "Stop after typing."
+ )
+ val = _field("password")
+ self.assertEqual(val, "Pass1234!",
+ msg=f"password='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+class TestEmailInput(_HermesFormBase):
+ """<input type=email> — fill with an email address."""
+
+ def test_email_input(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari'. "
+ "Then AX-click the Email input (id='f-email') and type '[email protected]'. "
+ "Stop after typing."
+ )
+ val = _field("email")
+ self.assertEqual(val, "[email protected]",
+ msg=f"email='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+class TestNumberInput(_HermesFormBase):
+ """<input type=number> — fill with a number."""
+
+ def test_number_input(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari'. "
+ "Then AX-click the Number input (id='f-number') and type '42'. "
+ "Stop after typing."
+ )
+ val = _field("number")
+ self.assertEqual(val, "42",
+ msg=f"number='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+class TestTextarea(_HermesFormBase):
+ """<textarea> — fill with a sentence."""
+
+ def test_textarea(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari'. "
+ "Then AX-click the Message textarea (id='f-textarea') and type 'bg works'. "
+ "Stop after typing."
+ )
+ val = _field("textarea")
+ self.assertIn("bg works", val,
+ msg=f"textarea='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+class TestCheckbox(_HermesFormBase):
+ """<input type=checkbox> — check it."""
+
+ def test_checkbox(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari'. "
+ "Then AX-click the checkbox labelled 'I agree to the terms' (id='f-checkbox'). "
+ "Stop after clicking."
+ )
+ self.assertTrue(_checkbox_checked(),
+ msg=f"checkbox not checked\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+class TestSelectDropdown(_HermesFormBase):
+ """<select> — pick an option from the dropdown.
+
+ Strategy: click the AXPopUpButton (which now returns available options +
+ a hint to use set_value), then call set_value with 'Blue'. The set_value
+ tool finds the 'Blue' child option and AXPresses it directly without
+ opening the native macOS popup menu.
+ """
+
+ def test_select_dropdown(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari' to see the AX tree. "
+ "Find the element for 'Favourite colour (Select)' (an AXPopUpButton, id='f-select'). "
+ "Use computer_use: action='set_value', element=<index>, value='Blue' to select "
+ "the Blue option directly without opening the native menu. Stop after setting.",
+ max_turns=4,
+ timeout=180,
+ )
+ val = _field("select")
+ self.assertEqual(val, "blue",
+ msg=f"select='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+class TestRadioButton(_HermesFormBase):
+ """<input type=radio> — select the Banana option."""
+
+ def test_radio_button(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari'. "
+ "Then AX-click the 'Banana' radio button (id='r-banana'). "
+ "Stop after clicking."
+ )
+ val = _field("radio")
+ self.assertEqual(val, "banana",
+ msg=f"radio='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+class TestSubmitButton(_HermesFormBase):
+ """<button type=submit> — click Submit and confirm the page responds."""
+
+ def test_submit_button(self) -> None:
+ out = self._run_hermes(
+ "Safari is in the background with a test form open. "
+ "Use computer_use: action='capture', app='Safari' to see the AX tree. "
+ "Find the AXButton labelled 'Submit Form' (id='submit-btn'). "
+ "Click it using its element index. Stop after clicking."
+ )
+ submitted = _js("(window._submitted !== undefined).toString()")
+ self.assertEqual(submitted, "true",
+ msg=f"form not submitted\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss()
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/libs/cua-driver/Tests/integration/test_overlay_z_order.py b/libs/cua-driver/Tests/integration/test_overlay_z_order.py
new file mode 100644
index 0000000000..ea48239e92
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/test_overlay_z_order.py
@@ -0,0 +1,213 @@
+"""Integration test: agent-cursor overlay z-ordering.
+
+Verifies that after driving a backgrounded window the cua-driver overlay
+is z-ordered JUST ABOVE the target window (NSWindowLevel.normal + ordered
+above target) rather than at NSWindowLevel.floating (above ALL normal windows).
+
+Expected ordering after pinning above a background window:
+ [... , target-window , overlay , windows-that-were-above-target , ...]
+
+Two assertions are verified:
+
+1. **overlay appears at layer 0** — the overlay's CGWindowLayer is 0
+ (NSWindowLevel.normal), so it shows up in list_windows. At .floating
+ (level 3) it would be absent from the layer-0 filtered list.
+
+2. **sandwich ordering** — overlay z_index > target z_index, and any
+ window that was above the target BEFORE the click remains above the
+ overlay AFTER the click.
+
+Run:
+ CUA_DRIVER_BINARY=.build/CuaDriver.app/Contents/MacOS/cua-driver \\
+ python3 -m unittest test_overlay_z_order -v
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from driver_client import DriverClient, default_binary_path, resolve_window_id # noqa: E402
+
+_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))
+_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp")
+_FOCUS_APP_EXE = os.path.join(
+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"
+)
+
+CALCULATOR_BUNDLE = "com.apple.calculator"
+
+
+def _ensure_focus_app_built() -> None:
+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")
+ if not os.path.exists(_FOCUS_APP_EXE) or (
+ os.path.exists(src) and os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)
+ ):
+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)
+
+
+def _launch_focus_app() -> tuple[subprocess.Popen, int]:
+ proc = subprocess.Popen(
+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True
+ )
+ for _ in range(40):
+ line = proc.stdout.readline().strip()
+ if line.startswith("FOCUS_PID="):
+ return proc, int(line.split("=", 1)[1])
+ time.sleep(0.1)
+ proc.terminate()
+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")
+
+
+class TestOverlayZOrder(unittest.TestCase):
+ """Overlay is sandwiched just above the target window, not floating above all."""
+
+ _focus_proc: subprocess.Popen
+ _focus_pid: int
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ _ensure_focus_app_built()
+ # Kill Calculator if stale
+ subprocess.run(["pkill", "-x", "Calculator"], check=False)
+ time.sleep(0.3)
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ subprocess.run(["pkill", "-x", "Calculator"], check=False)
+
+ def setUp(self) -> None:
+ self.client = DriverClient(default_binary_path()).__enter__()
+ # Ensure agent cursor is enabled for this test session.
+ self.client.call_tool("set_agent_cursor_enabled", {"enabled": True})
+
+ # Launch Calculator in background (it may briefly become frontmost).
+ r = self.client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE})
+ self.calc_pid = r["structuredContent"]["pid"]
+ time.sleep(0.5)
+
+ # Launch FocusMonitorApp as frontmost so Calculator is definitely behind it.
+ self._focus_proc, self._focus_pid = _launch_focus_app()
+ time.sleep(0.5) # let it settle as the frontmost window
+
+ def tearDown(self) -> None:
+ self._focus_proc.terminate()
+ subprocess.run(["pkill", "-x", "Calculator"], check=False)
+ self.client.__exit__(None, None, None)
+
+ # ------------------------------------------------------------------
+
+ def test_overlay_at_normal_level_above_target(self) -> None:
+ """Overlay appears at layer=0 (not .floating/layer=3) and z_index > target."""
+ # Snapshot all on-screen windows BEFORE the click.
+ all_before = self._all_on_screen_windows()
+ driver_pid = self.client.process.pid
+
+ calc_wins_before = [w for w in all_before if w["pid"] == self.calc_pid]
+ self.assertTrue(calc_wins_before, "Calculator window not visible before click")
+ calc_z_before = max(w["z_index"] for w in calc_wins_before)
+ calc_win_before = max(calc_wins_before, key=lambda w: w["z_index"])
+ calc_win_id = calc_win_before["window_id"]
+
+ # Use a pixel-addressed click at the center of the Calculator window
+ # so we don't need get_window_state element indices.
+ b = calc_win_before["bounds"]
+ cx = b["width"] / 2.0
+ cy = b["height"] / 2.0
+
+ # Windows above Calculator that are NOT the driver overlay.
+ fg_wins_before = [
+ w for w in all_before
+ if w["z_index"] > calc_z_before
+ and w["pid"] != driver_pid
+ and w.get("is_on_screen")
+ ]
+
+ # Trigger a pixel click to make the overlay appear and pin above Calculator.
+ self.client.call_tool(
+ "click",
+ {
+ "pid": self.calc_pid,
+ "window_id": calc_win_id,
+ "x": cx,
+ "y": cy,
+ },
+ )
+
+ # Let the defensive-repin ticks fully settle (last tick fires at ~1200ms
+ # after the second pinAbove inside click, which returns after animations).
+ time.sleep(1.5)
+
+ # ── Assertion 1: overlay is visible at layer 0 ─────────────────────────
+ all_after = self._all_on_screen_windows()
+ overlay_wins = [w for w in all_after if w["pid"] == driver_pid]
+ self.assertTrue(
+ overlay_wins,
+ f"cua-driver overlay NOT visible in list_windows at layer=0 "
+ f"(driver pid={driver_pid}). "
+ f"Expected NSWindowLevel.normal overlay to appear as a layer-0 window. "
+ f"If level is still .floating (layer 3) it would be filtered out here.",
+ )
+
+ overlay_z = max(w["z_index"] for w in overlay_wins)
+
+ # ── Assertion 2: overlay is above the target ────────────────────────────
+ calc_wins_after = [w for w in all_after if w["pid"] == self.calc_pid]
+ calc_z_after = (
+ max(w["z_index"] for w in calc_wins_after) if calc_wins_after else 0
+ )
+ self.assertGreater(
+ overlay_z,
+ calc_z_after,
+ f"overlay (z={overlay_z}) must be ABOVE Calculator (z={calc_z_after}). "
+ f"pinAbove must order the overlay at z_target+1.",
+ )
+
+ # ── Assertion 3: foreground windows remain above the overlay ────────────
+ # Any window that was above Calculator BEFORE the click and is still
+ # on-screen AFTER the click must have z_index > overlay_z.
+ # This is the key invariant that distinguishes .normal from .floating.
+ missed_above = []
+ for w_before in fg_wins_before:
+ w_after = next(
+ (a for a in all_after if a["window_id"] == w_before["window_id"]),
+ None,
+ )
+ if w_after and w_after.get("is_on_screen"):
+ if w_after["z_index"] <= overlay_z:
+ missed_above.append(
+ f"{w_before['app_name']} "
+ f"(window_id={w_before['window_id']}, "
+ f"z_before={w_before['z_index']}, "
+ f"z_after={w_after['z_index']}, overlay_z={overlay_z})"
+ )
+
+ if fg_wins_before and missed_above:
+ self.fail(
+ "Window(s) that were ABOVE Calculator before the click are now "
+ "BELOW OR EQUAL TO the overlay — overlay is NOT sandwiched:\n"
+ + "\n".join(f" {m}" for m in missed_above)
+ + "\nExpected ordering: [target, overlay, fg-windows]. "
+ "This means the overlay is still at .floating level instead of .normal."
+ )
+
+ # If there were no foreground windows above Calculator, just note it.
+ if not fg_wins_before:
+ # Calculator was already frontmost; z-sandwich can't be verified.
+ # Assertion 1 + 2 are sufficient in this case.
+ pass
+
+ # ------------------------------------------------------------------
+
+ def _all_on_screen_windows(self) -> list[dict]:
+ result = self.client.call_tool("list_windows", {"on_screen_only": True})
+ return result["structuredContent"]["windows"]
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)