Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): Chrome CGEvent modifier leak, silent-accept detection, cursor animation (#1422)

ddupontff0a6478c910
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Input/KeyboardInput.swift b/libs/cua-driver/Sources/CuaDriverCore/Input/KeyboardInput.swift
index 4e7055153b..0f9b712e75 100644
--- a/libs/cua-driver/Sources/CuaDriverCore/Input/KeyboardInput.swift
+++ b/libs/cua-driver/Sources/CuaDriverCore/Input/KeyboardInput.swift
@@ -138,6 +138,14 @@ public enum KeyboardInput {
/// is left at 0 — macOS uses the attached unicode string instead.
/// Pass `toPid` to deliver the events directly to a process's queue
/// rather than the system HID tap.
+ ///
+ /// **Modifier-state isolation**: when posting via `postToPid` some
+ /// apps (notably Chrome) inspect the CGEvent's `flags` field to infer
+ /// whether Shift or other modifiers were held to produce the unicode
+ /// character. If `flags` is non-zero the app may "remember" the
+ /// modifier and apply it to the next character. We explicitly clear
+ /// `event.flags` on every event so the receiver always sees a clean
+ /// modifier state regardless of what was typed before.
private static func sendUnicodeCharacter(
_ character: Character, toPid pid: pid_t? = nil
) throws {
@@ -154,6 +162,11 @@ public enum KeyboardInput {
"unicode character \"\(character)\" down=\(keyDown)"
)
}
+ // Clear all modifier flags so the receiver doesn't accumulate
+ // Shift/Control/Option state between characters. Without this,
+ // Chrome interprets uppercase unicode events as "Shift held" and
+ // applies Shift to the next character (e.g. 'E' + '2' → '@').
+ event.flags = []
utf16.withUnsafeBufferPointer { buffer in
if let base = buffer.baseAddress {
event.keyboardSetUnicodeString(
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift
index d69112f834..d33473736c 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift
@@ -80,6 +80,15 @@ public enum SetValueTool {
)
let target = AXInput.describe(element)
+ // Animate the agent cursor to the target element before
+ // setting its value — same visual feedback as click.
+ // No-op when the cursor is disabled or the element has no
+ // resolvable screen position.
+ if let center = AXInput.screenCenter(of: element) {
+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }
+ await AgentCursor.shared.animateAndWait(to: center)
+ }
+
// ── AXPopUpButton (HTML <select>) special path ──────────
// Safari WebKit does not propagate AXValue writes back to
// the HTML DOM for <select> elements — the AX write returns
@@ -88,32 +97,39 @@ public enum SetValueTool {
// AXPress the one whose AXTitle or AXValue matches `value`.
// This bypasses the native macOS popup menu entirely, so the
// option is selected without the menu needing to be visible.
+ let result: CallTool.Result
if target.role == "AXPopUpButton" {
- return try await selectPopupOption(
+ result = try await selectPopupOption(
element: element,
index: index,
pid: pid,
value: value,
elementTitle: target.title ?? ""
)
- }
-
- // ── Default path: write AXValue directly ────────────────
- try await AppStateRegistry.focusGuard.withFocusSuppressed(
- pid: pid,
- element: element
- ) {
- try AXInput.setAttribute(
- "AXValue",
- on: element,
- value: value as CFTypeRef
+ } else {
+ // ── Default path: write AXValue directly ────────────────
+ try await AppStateRegistry.focusGuard.withFocusSuppressed(
+ pid: pid,
+ element: element
+ ) {
+ try AXInput.setAttribute(
+ "AXValue",
+ on: element,
+ value: value as CFTypeRef
+ )
+ }
+ let summary =
+ "✅ Set AXValue on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"."
+ result = CallTool.Result(
+ content: [.text(text: summary, annotations: nil, _meta: nil)]
)
}
- let summary =
- "✅ Set AXValue on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"."
- return CallTool.Result(
- content: [.text(text: summary, annotations: nil, _meta: nil)]
- )
+ // Cursor press-pulse after the action — mirrors ClickTool's
+ // finishClick sequence. No-op when the cursor is disabled.
+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }
+ await AgentCursor.shared.playClickPress()
+ await AgentCursor.shared.finishClick(pid: pid)
+ return result
} catch let error as AppStateError {
return errorResult(error.description)
} catch let error as AXInputError {
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swift
index 16035c3599..084d54ae55 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swift
@@ -99,9 +99,11 @@ public enum TypeTextTool {
+ "the same window_id you used in `get_window_state`.")
}
- // Attempt AX bulk-insert. On any AXInputError fall back to
- // CGEvent character synthesis, which reaches Chromium / Electron
- // inputs that don't expose kAXSelectedText.
+ // Attempt AX bulk-insert. On any AXInputError — or when a
+ // silent-accept-but-no-DOM-update is detected (Chromium web
+ // inputs accept kAXSelectedText writes without error but never
+ // propagate the value to the HTML DOM) — fall back to CGEvent
+ // character synthesis.
do {
if let index = elementIndex, let rawWindowId {
guard let windowId = UInt32(exactly: rawWindowId) else {
@@ -112,6 +114,13 @@ public enum TypeTextTool {
pid: pid,
windowId: windowId,
elementIndex: index)
+ // Animate the agent cursor to the target element before
+ // writing — same visual feedback as click. No-op when the
+ // cursor is disabled or the element has no screen position.
+ if let center = AXInput.screenCenter(of: element) {
+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }
+ await AgentCursor.shared.animateAndWait(to: center)
+ }
do {
try await AppStateRegistry.focusGuard.withFocusSuppressed(
pid: pid, element: element
@@ -122,42 +131,78 @@ public enum TypeTextTool {
value: text as CFTypeRef
)
}
- let target = AXInput.describe(element)
- let summary =
- "✅ Inserted \(text.count) char(s) into [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\" on pid \(rawPid) via AX."
- return CallTool.Result(
- content: [.text(text: summary, annotations: nil, _meta: nil)]
- )
+ // Verify the write was actually applied — Chromium
+ // web inputs silently accept the call but never
+ // update their DOM value. Read AXValue back; if it's
+ // still empty when we just wrote non-empty text the
+ // write was a no-op and we must use CGEvent instead.
+ if !text.isEmpty,
+ let axValue = AXInput.stringAttribute("AXValue", of: element),
+ axValue.isEmpty
+ {
+ // Silent-accept detected — fall through.
+ } else {
+ let target = AXInput.describe(element)
+ let summary =
+ "✅ Inserted \(text.count) char(s) into [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\" on pid \(rawPid) via AX."
+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }
+ await AgentCursor.shared.playClickPress()
+ await AgentCursor.shared.finishClick(pid: pid)
+ return CallTool.Result(
+ content: [.text(text: summary, annotations: nil, _meta: nil)]
+ )
+ }
} catch is AXInputError {
// AX rejected — fall through to CGEvent synthesis.
}
} else {
do {
let element = try AXInput.focusedElement(pid: pid)
+ // Animate the cursor to the focused element before writing.
+ if let center = AXInput.screenCenter(of: element) {
+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }
+ await AgentCursor.shared.animateAndWait(to: center)
+ }
try AXInput.setAttribute(
"AXSelectedText",
on: element,
value: text as CFTypeRef
)
- let target = AXInput.describe(element)
- let summary =
- "✅ Inserted \(text.count) char(s) into focused \(target.role ?? "?") \"\(target.title ?? "")\" on pid \(rawPid) via AX."
- return CallTool.Result(
- content: [.text(text: summary, annotations: nil, _meta: nil)]
- )
+ // Same silent-accept check as the element_index path.
+ if !text.isEmpty,
+ let axValue = AXInput.stringAttribute("AXValue", of: element),
+ axValue.isEmpty
+ {
+ // Silent-accept detected — fall through.
+ } else {
+ let target = AXInput.describe(element)
+ let summary =
+ "✅ Inserted \(text.count) char(s) into focused \(target.role ?? "?") \"\(target.title ?? "")\" on pid \(rawPid) via AX."
+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }
+ await AgentCursor.shared.playClickPress()
+ await AgentCursor.shared.finishClick(pid: pid)
+ return CallTool.Result(
+ content: [.text(text: summary, annotations: nil, _meta: nil)]
+ )
+ }
} catch is AXInputError {
// AX rejected — fall through to CGEvent synthesis.
}
}
- // CGEvent fallback path.
+ // CGEvent fallback path — reached when AX write was rejected
+ // (AXInputError) or silently accepted without updating the
+ // DOM value (Chromium web inputs).
try KeyboardInput.typeCharacters(
text,
delayMilliseconds: delayMs,
toPid: pid
)
let summary =
- "✅ Typed \(text.count) char(s) on pid \(rawPid) via CGEvent (AX fallback, \(delayMs)ms delay)."
+ "✅ Typed \(text.count) char(s) on pid \(rawPid) via CGEvent (\(delayMs)ms delay)."
+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }
+ await AgentCursor.shared.playClickPress()
+ await AgentCursor.shared.finishClick(pid: pid)
return CallTool.Result(
content: [.text(text: summary, annotations: nil, _meta: nil)]
)
diff --git a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
index 44225353a9..74ae4addfe 100644
--- a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
+++ b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
@@ -32,7 +32,7 @@ button#submit-btn:hover { background: #0066DD; }
</head>
<body>
<h1>All Inputs Test Form</h1>
-<form id="test-form" onsubmit="handleSubmit(event)">
+<form id="test-form" onsubmit="handleSubmit(event)" autocomplete="off">
<div class="field">
<label for="f-text">Text</label>
diff --git a/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py b/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py
new file mode 100644
index 0000000000..2ddff05cc8
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py
@@ -0,0 +1,603 @@
+"""Integration test: hermes CLI drives Chrome form fields in the background.
+
+Goal
+----
+One test per HTML input type. Each test asks hermes to interact with exactly
+ONE element in a Chrome window that is NOT frontmost — verifying that:
+
+ 1. cua-driver delivers events to Chromium inputs without stealing focus.
+ The unified ``type_text`` tool automatically falls back to CGEvent
+ synthesis when its AX attribute-write path is rejected by Chrome's
+ web-content inputs — callers (hermes) don't need to know which path ran.
+
+ 2. ``launch_app`` (not ``open -a``) is used to keep Chrome hidden and
+ avoid the self-activation race that ``open -g`` suffers from.
+
+ 3. FocusMonitorApp retains the frontmost / key-window slot throughout.
+
+Chrome vs. Safari differences
+------------------------------
+* ``type_text`` always takes the CGEvent fallback for Chromium inputs;
+ the AX attribute-write path is silently rejected and the response says
+ "via CGEvent" rather than "via AX".
+* Field-value verification uses osascript against Chrome's AppleScript
+ interface ("Allow JavaScript from Apple Events" must be enabled —
+ setUpClass patches the Preferences JSON automatically).
+* Setup uses ``launch_app`` (driver-level), not ``open -g``.
+
+Setup (setUpClass)
+------------------
+1. Kill Chrome, enable "Allow JavaScript from Apple Events" (Preferences
+ JSON patch), relaunch Chrome with the form URL via cua-driver's
+ ``launch_app`` tool so it stays hidden.
+2. Wait for the form's ``getFieldValues`` helper to be callable.
+3. Launch FocusMonitorApp last — it becomes frontmost and holds focus for
+ every test in the class.
+
+Per-test design
+---------------
+* Prompt is intentionally minimal: capture Chrome, interact with ONE named
+ element, stop. We do NOT ask hermes to re-capture or verify — that keeps
+ tool-call count to ≤ 4.
+* After hermes returns the test reads the field value via osascript and
+ asserts the expected value was written.
+* The FocusMonitorApp focus-loss counters must not increase.
+
+Run
+---
+ scripts/test.sh test_hermes_chrome_form_fill
+ python3 -m pytest Tests/integration/test_hermes_chrome_form_fill.py -v
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from driver_client import DriverClient, default_binary_path, resolve_window_id # noqa: E402
+
+# ---------------------------------------------------------------------------
+# Paths & constants
+# ---------------------------------------------------------------------------
+
+_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))
+
+_HTML_FORM = os.path.join(_THIS_DIR, "fixtures", "form_all_inputs.html")
+_FORM_URL = f"file://{_HTML_FORM}"
+
+_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp")
+_FOCUS_APP_EXE = os.path.join(
+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"
+)
+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"
+_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt"
+_FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt"
+
+_HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes")
+
+CHROME_BUNDLE = "com.google.Chrome"
+
+_ANTHROPIC_KEY: str = os.environ.get("ANTHROPIC_API_KEY", "")
+
+# Use haiku for speed/cost; override with HERMES_TEST_MODEL for a stronger model.
+_MODEL = os.environ.get("HERMES_TEST_MODEL", "claude-haiku-4-5-20251001")
+
+
+# ---------------------------------------------------------------------------
+# Chrome Apple Events JS helpers
+# ---------------------------------------------------------------------------
+
+def _enable_chrome_apple_events() -> None:
+ """Quit Chrome, write allow_javascript_apple_events to all profiles.
+
+ Mirrors the same helper in test_browser_js.py — patches Preferences JSON
+ directly so we don't depend on the ``page`` tool's ``enable_javascript_apple_events``
+ action (which itself needs Apple Events to already be on for the confirmation).
+ Does NOT relaunch Chrome — the caller launches it via ``launch_app`` afterwards.
+ """
+ subprocess.run(
+ ["osascript", "-e", 'quit app "Google Chrome"'],
+ check=False, timeout=10,
+ )
+ time.sleep(1.5)
+
+ for prefs_path in glob.glob(
+ os.path.expanduser(
+ "~/Library/Application Support/Google/Chrome/*/Preferences"
+ )
+ ):
+ profile = prefs_path.split("/")[-2]
+ if "System" in profile or "Guest" in profile:
+ continue
+ try:
+ with open(prefs_path) as f:
+ data = json.load(f)
+ data.setdefault("browser", {})["allow_javascript_apple_events"] = True
+ data.setdefault("account_values", {}).setdefault("browser", {})[
+ "allow_javascript_apple_events"
+ ] = True
+ with open(prefs_path, "w") as f:
+ json.dump(data, f)
+ print(f" Patched {profile}/Preferences")
+ except Exception as e:
+ print(f" Skipped {profile}: {e}")
+
+
+def _js_chrome(expr: str) -> str:
+ """Evaluate a JS expression in Chrome's front tab via osascript.
+
+ Uses json.dumps to safely quote the expression string for AppleScript.
+ Requires "Allow JavaScript from Apple Events" to be enabled in Chrome.
+ Returns stdout stripped of trailing whitespace.
+ """
+ # AppleScript: tell application "Google Chrome" to execute front window's
+ # active tab javascript "<expr>"
+ osa = (
+ "tell application \"Google Chrome\" to execute "
+ f"front window's active tab javascript {json.dumps(expr)}"
+ )
+ r = subprocess.run(
+ ["osascript", "-e", osa], capture_output=True, text=True, timeout=15
+ )
+ return r.stdout.strip()
+
+
+def _wait_for_chrome_form(timeout: float = 30.0) -> bool:
+ """Wait until Chrome's active tab has the test form's key element loaded.
+
+ Chrome's AppleScript 'execute javascript' runs in an isolated world —
+ page-defined globals like getFieldValues() are NOT visible. We probe for
+ the DOM element instead (document.getElementById is available in any world).
+ """
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ result = _js_chrome('document.getElementById("f-text") ? "ready" : "not-ready"')
+ if result == "ready":
+ return True
+ time.sleep(0.5)
+ return False
+
+
+# Map logical field names → HTML element IDs in form_all_inputs.html.
+_FIELD_TO_ID: dict[str, str] = {
+ "text": "f-text",
+ "email": "f-email",
+ "password": "f-password",
+ "number": "f-number",
+ "tel": "f-tel",
+ "textarea": "f-textarea",
+ "select": "f-select",
+ "range": "f-range",
+ "date": "f-date",
+ "color": "f-color",
+}
+
+
+def _field(name: str) -> str:
+ """Read a single form field value from Chrome.
+
+ Chrome's AppleScript 'execute javascript' runs in an isolated world, so
+ page-defined helpers like getFieldValues() are not visible. We read the
+ DOM element's .value property directly instead.
+ """
+ elem_id = _FIELD_TO_ID.get(name, f"f-{name}")
+ val = _js_chrome(
+ f'(function(){{var e=document.getElementById("{elem_id}");'
+ f'return e ? e.value : "__missing__";}})()'
+ )
+ return val if val != "__missing__" else ""
+
+
+def _checkbox_checked() -> bool:
+ val = _js_chrome(
+ '(function(){var e=document.getElementById("f-checkbox");'
+ 'return e ? e.checked.toString() : "false";})()'
+ )
+ return val.lower() == "true"
+
+
+# ---------------------------------------------------------------------------
+# FocusMonitorApp helpers
+# ---------------------------------------------------------------------------
+
+def _launch_focus_app() -> tuple[subprocess.Popen, int]:
+ proc = subprocess.Popen(
+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
+ )
+ for _ in range(40):
+ line = proc.stdout.readline().strip()
+ if line.startswith("FOCUS_PID="):
+ return proc, int(line.split("=", 1)[1])
+ time.sleep(0.1)
+ proc.terminate()
+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")
+
+
+def _read_file_int(path: str) -> int:
+ try:
+ with open(path) as f:
+ return int(f.read().strip())
+ except (FileNotFoundError, ValueError):
+ return 0
+
+
+# ---------------------------------------------------------------------------
+# Base test class
+# ---------------------------------------------------------------------------
+
+class _HermesChromFormBase(unittest.TestCase):
+ """Shared setup/teardown and hermes runner for all Chrome per-input tests."""
+
+ _focus_proc: subprocess.Popen
+ _focus_pid: int
+ _chrome_pid: int
+ binary: str
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ if not _ANTHROPIC_KEY:
+ raise unittest.SkipTest(
+ "ANTHROPIC_API_KEY is not set — skipping hermes Chrome form-fill tests"
+ )
+ if not os.path.exists(_HERMES_BIN):
+ raise unittest.SkipTest(
+ f"hermes not found at {_HERMES_BIN} — install hermes first"
+ )
+
+ for f in (_LOSS_FILE, _KEY_LOSS_FILE, _FIELD_LOSS_FILE):
+ try:
+ os.remove(f)
+ except FileNotFoundError:
+ pass
+
+ # Rebuild FocusMonitorApp when Swift source is newer.
+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")
+ if (not os.path.exists(_FOCUS_APP_EXE)
+ or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)):
+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)
+
+ cls.binary = default_binary_path()
+
+ # Step 1: Enable "Allow JavaScript from Apple Events" in Chrome.
+ # This quits Chrome, patches every profile's Preferences JSON.
+ print("\n Enabling Chrome Apple Events JS...")
+ _enable_chrome_apple_events()
+
+ # Step 2: Launch Chrome hidden with the form URL via cua-driver's
+ # launch_app — no activation, no window raise, no focus steal.
+ print(f" Launching Chrome hidden with {_FORM_URL}")
+ with DriverClient(cls.binary) as c:
+ result = c.call_tool("launch_app", {
+ "bundle_id": CHROME_BUNDLE,
+ "urls": [_FORM_URL],
+ })
+ sc = result.get("structuredContent", {})
+ cls._chrome_pid = sc.get("pid", 0)
+ if not cls._chrome_pid:
+ # Fallback: extract from list_apps
+ apps = c.call_tool("list_apps")["structuredContent"]["apps"]
+ for app in apps:
+ if app.get("bundle_id") == CHROME_BUNDLE:
+ cls._chrome_pid = app["pid"]
+ break
+ print(f" Chrome pid: {cls._chrome_pid}")
+
+ # Step 3: Wait for Chrome to load the form and expose getFieldValues().
+ if not _wait_for_chrome_form(30.0):
+ raise RuntimeError(
+ f"Chrome did not load {_HTML_FORM} within 30 s.\n"
+ "Check: (1) file exists, (2) 'Allow JavaScript from Apple Events' "
+ "is enabled in Chrome (Develop menu)."
+ )
+ print(" Form loaded — f-text element ready")
+
+ # Step 4: Launch FocusMonitorApp last — it becomes frontmost and holds
+ # the focus token for every test.
+ cls._focus_proc, cls._focus_pid = _launch_focus_app()
+ print(f" FocusMonitorApp pid: {cls._focus_pid}")
+ # Wait for FocusMonitorApp's focus-loss counter to stabilize.
+ # The field loss counter bumps once during startup (macOS transitions
+ # first-responder during the window-becoming-key sequence). We wait
+ # until the counter hasn't changed for 0.5 s before recording the
+ # baseline in setUp(), so startup-phase transients are not counted
+ # as test failures.
+ prev = _read_file_int(_FIELD_LOSS_FILE)
+ for _ in range(20): # up to 10 s
+ time.sleep(0.5)
+ curr = _read_file_int(_FIELD_LOSS_FILE)
+ if curr == prev:
+ break
+ prev = curr
+ print(f" FocusMonitorApp ready (field_losses baseline: {_read_file_int(_FIELD_LOSS_FILE)})")
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ cls._focus_proc.terminate()
+ try:
+ cls._focus_proc.wait(timeout=3)
+ except subprocess.TimeoutExpired:
+ cls._focus_proc.kill()
+ subprocess.run(
+ ["osascript", "-e", 'quit app "Google Chrome"'],
+ check=False, timeout=10,
+ )
+
+ def setUp(self) -> None:
+ self._losses_before = _read_file_int(_LOSS_FILE)
+ self._key_losses_before = _read_file_int(_KEY_LOSS_FILE)
+ self._field_losses_before = _read_file_int(_FIELD_LOSS_FILE)
+
+ def _run_hermes(self, prompt: str, timeout: int = 120, max_turns: int = 5) -> str:
+ """Run hermes with the anthropic provider and return combined stdout+stderr."""
+ env = os.environ.copy()
+ env["ANTHROPIC_API_KEY"] = _ANTHROPIC_KEY
+ env["HERMES_COMPUTER_USE_BACKEND"] = "cua"
+ env["HERMES_CUA_DRIVER_CMD"] = default_binary_path()
+
+ result = subprocess.run(
+ [
+ _HERMES_BIN, "chat",
+ "--provider", "anthropic",
+ "-m", _MODEL,
+ "--toolsets", "computer_use",
+ "--yolo",
+ "-Q",
+ "--max-turns", str(max_turns),
+ "-q", prompt,
+ ],
+ env=env, capture_output=True, text=True, timeout=timeout,
+ )
+ return (result.stdout + "\n" + result.stderr).strip()
+
+ def _assert_no_new_focus_loss(self, allow_field_losses: int = 0) -> None:
+ """Assert no new app-level or key-level focus losses occurred.
+
+ ``allow_field_losses`` controls how many text-field first-responder
+ losses are tolerated. Chrome interactions trigger one field loss per
+ AX action via a known ``SyntheticAppFocusEnforcer`` side effect:
+ writing ``AXFocused=true`` on Chrome's window emits an AX
+ ``kAXFocusedWindowChangedNotification`` that briefly disrupts
+ FocusMonitorApp's first-responder chain. App-level and key-window
+ focus are NOT affected (Chrome never becomes frontmost). Pass
+ ``allow_field_losses=1`` for single-action Chrome tests or a higher
+ value for multi-action tests.
+ """
+ time.sleep(0.3)
+ app_delta = _read_file_int(_LOSS_FILE) - self._losses_before
+ key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before
+ field_delta = _read_file_int(_FIELD_LOSS_FILE) - self._field_losses_before
+ msgs = []
+ if app_delta:
+ msgs.append(f"app lost focus {app_delta}x")
+ if key_delta:
+ msgs.append(f"window lost key status {key_delta}x")
+ if field_delta > allow_field_losses:
+ msgs.append(
+ f"text-input lost first-responder {field_delta}x"
+ f" (allowed: {allow_field_losses})"
+ )
+ self.assertEqual(
+ len(msgs), 0,
+ f"Focus stolen in {self._testMethodName}: "
+ + "; ".join(msgs)
+ + " — background CGEvent delivery must not steal focus",
+ )
+
+ def _chrome_window_id(self) -> int:
+ """Resolve Chrome's current on-screen window_id for AX verification."""
+ with DriverClient(self.binary) as c:
+ return resolve_window_id(c, self._chrome_pid)
+
+ def _ax_contains(self, query: str) -> bool:
+ """True if Chrome's AX tree contains a line matching *query* (case-insensitive).
+
+ Uses get_window_state's query= filter so we only check the relevant
+ subtree — avoids false positives from unrelated UI text.
+ """
+ try:
+ wid = self._chrome_window_id()
+ with DriverClient(self.binary) as c:
+ snap = c.call_tool("get_window_state", {
+ "pid": self._chrome_pid,
+ "window_id": wid,
+ "query": query,
+ })
+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")
+ return len(tree.strip()) > 0
+ except Exception:
+ return False
+
+
+# ---------------------------------------------------------------------------
+# One test class per input type
+# ---------------------------------------------------------------------------
+
+class TestChromeTextInput(_HermesChromFormBase):
+ """<input type=text> in Chrome — type_text CGEvent fallback path."""
+
+ def test_text_input(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form open. "
+ "Use computer_use: action='capture', app='Google Chrome' to see the form. "
+ "Click the Text input (id='f-text') to focus it, "
+ "then type 'Hello Chrome' into it. Stop after typing."
+ )
+ val = _field("text")
+ self.assertEqual(val, "Hello Chrome",
+ msg=f"text='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+class TestChromeEmailInput(_HermesChromFormBase):
+ """<input type=email> in Chrome — CGEvent fallback for Chromium inputs."""
+
+ def test_email_input(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form open. "
+ "Use computer_use: action='capture', app='Google Chrome'. "
+ "Click the Email input (id='f-email') and type '[email protected]'. "
+ "Stop after typing."
+ )
+ val = _field("email")
+ self.assertEqual(val, "[email protected]",
+ msg=f"email='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+class TestChromePasswordInput(_HermesChromFormBase):
+ """<input type=password> in Chrome."""
+
+ def test_password_input(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form open. "
+ "Use computer_use: action='capture', app='Google Chrome'. "
+ "Click the Password input (id='f-password') and type 'Secure123!'. "
+ "Stop after typing."
+ )
+ val = _field("password")
+ self.assertEqual(val, "Secure123!",
+ msg=f"password='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+class TestChromeNumberInput(_HermesChromFormBase):
+ """<input type=number> in Chrome."""
+
+ def test_number_input(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form open. "
+ "Use computer_use: action='capture', app='Google Chrome'. "
+ "Click the Number input (id='f-number') and type '99'. "
+ "Stop after typing."
+ )
+ val = _field("number")
+ self.assertEqual(val, "99",
+ msg=f"number='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+class TestChromeTextarea(_HermesChromFormBase):
+ """<textarea> in Chrome."""
+
+ def test_textarea(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form open. "
+ "Use computer_use: action='capture', app='Google Chrome'. "
+ "Click the Message textarea (id='f-textarea') and type 'chrome bg ok'. "
+ "Stop after typing."
+ )
+ val = _field("textarea")
+ self.assertIn("chrome bg ok", val,
+ msg=f"textarea='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+class TestChromeCheckbox(_HermesChromFormBase):
+ """<input type=checkbox> in Chrome — AX click."""
+
+ def test_checkbox(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form open. "
+ "Use computer_use: action='capture', app='Google Chrome'. "
+ "Click the checkbox labelled 'I agree to the terms' (id='f-checkbox'). "
+ "Stop after clicking."
+ )
+ self.assertTrue(_checkbox_checked(),
+ msg=f"checkbox not checked\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+class TestChromeSelectDropdown(_HermesChromFormBase):
+ """<select> in Chrome — set_value on AXPopUpButton.
+
+ Chrome exposes HTML <select> as AXPopUpButton. The set_value tool finds
+ the matching option AXMenuItemMarkChar and AXPresses it directly without
+ opening the native macOS menu — no window raise, no focus steal.
+ """
+
+ def test_select_dropdown(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form open. "
+ "Use computer_use: action='capture', app='Google Chrome' to see the AX tree. "
+ "Find the element for 'Favourite colour (Select)' (an AXPopUpButton, id='f-select'). "
+ "Use computer_use: action='set_value', element=<index>, value='Blue' to select "
+ "the Blue option directly. Stop after setting.",
+ max_turns=4,
+ timeout=180,
+ )
+ val = _field("select")
+ self.assertEqual(val, "blue",
+ msg=f"select='{val}'\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+class TestChromeSubmitButton(_HermesChromFormBase):
+ """<button type=submit> in Chrome — click and verify form submission."""
+
+ def test_submit_button(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form open. "
+ "Use computer_use: action='capture', app='Google Chrome' to see the AX tree. "
+ "Find the AXButton labelled 'Submit Form' (id='submit-btn') and click it. "
+ "Stop after clicking."
+ )
+ # Chrome's AppleScript JS runs in an isolated world — check the DOM
+ # result element that the form's submit handler populates instead of
+ # the page-global window._submitted (which isn't visible here).
+ submitted = _js_chrome(
+ '(function(){var e=document.getElementById("result");'
+ 'return (e && e.textContent.trim().length > 0) ? "true" : "false";})()'
+ )
+ self.assertEqual(submitted, "true",
+ msg=f"form not submitted\nhermes: {out[-400:]}")
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+class TestChromeFullFormFlow(_HermesChromFormBase):
+ """End-to-end: fill text + email + number, then submit — all in background Chrome.
+
+ This is the main e2e smoke test confirming:
+ - launch_app correctly opens Chrome without activation
+ - type_text's CGEvent fallback writes into Chromium inputs
+ - Multiple tool calls across a single hermes session don't leak focus
+ """
+
+ def test_full_form_flow(self) -> None:
+ out = self._run_hermes(
+ "Google Chrome is in the background with a test HTML form at "
+ f"'{_FORM_URL}'. "
+ "Use computer_use to fill the following fields (capture Chrome first, "
+ "then click each field before typing):\n"
+ " - Text input (id='f-text'): type 'E2E Test'\n"
+ " - Email input (id='f-email'): type '[email protected]'\n"
+ " - Number input (id='f-number'): type '7'\n"
+ "After filling all three fields, stop. "
+ "Do NOT submit the form.",
+ max_turns=20,
+ timeout=360,
+ )
+ # Verify all three fields were written.
+ text_val = _field("text")
+ email_val = _field("email")
+ num_val = _field("number")
+ errors: list[str] = []
+ if text_val != "E2E Test": errors.append(f"text='{text_val}'")
+ if email_val != "[email protected]": errors.append(f"email='{email_val}'")
+ if num_val != "7": errors.append(f"number='{num_val}'")
+ self.assertFalse(
+ errors,
+ f"Field mismatch: {'; '.join(errors)}\nhermes: {out[-600:]}"
+ )
+ self._assert_no_new_focus_loss(allow_field_losses=3)
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py b/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py
new file mode 100644
index 0000000000..8239deb616
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py
@@ -0,0 +1,270 @@
+"""Integration test: hermes can launch Safari in the background via launch_app.
+
+What this test proves
+---------------------
+1. ``computer_use(action='launch_app', bundle_id='com.apple.Safari',
+ urls=['https://example.com'])`` actually creates a Safari process and
+ window — verified independently via a direct cua-driver MCP call to
+ ``list_windows``, bypassing any model hallucination.
+2. The launch does NOT steal focus from FocusMonitorApp: app-level focus,
+ key-window status, and text-field first-responder are all unchanged.
+3. ``capture(app='Safari')`` after launch returns Safari's AX tree and
+ screenshot, not whatever window happens to be frontmost (Finder, etc.).
+
+Key insight: Safari launched with ``hides=true`` and no URL never creates a
+window — the process starts but NSWorkspace never triggers window creation.
+Passing ``urls=['https://example.com']`` forces Safari to open a tab and
+create a window. The window IS on-screen (z-ordered behind other windows)
+but Safari is NOT the active app — that is the correct "background" behavior.
+
+Setup
+-----
+Safari is killed before the test so we know the process was freshly launched
+by hermes. FocusMonitorApp is launched last and holds focus throughout.
+
+Run
+---
+ python3 -m pytest libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py -v
+ HERMES_TEST_MODEL=claude-sonnet-4-6 python3 -m pytest ... -v
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from driver_client import DriverClient, default_binary_path # noqa: E402
+
+# ---------------------------------------------------------------------------
+# Paths & constants
+# ---------------------------------------------------------------------------
+
+_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))
+
+_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp")
+_FOCUS_APP_EXE = os.path.join(
+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"
+)
+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"
+_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt"
+_FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt"
+
+_HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes")
+
+_ANTHROPIC_KEY: str = os.environ.get("ANTHROPIC_API_KEY", "")
+_MODEL = os.environ.get("HERMES_TEST_MODEL", "claude-haiku-4-5-20251001")
+
+_SAFARI_BUNDLE = "com.apple.Safari"
+_LAUNCH_URL = "https://example.com"
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _read_file_int(path: str) -> int:
+ try:
+ with open(path) as f:
+ return int(f.read().strip())
+ except (FileNotFoundError, ValueError):
+ return 0
+
+
+def _safari_pids() -> list[int]:
+ r = subprocess.run(["pgrep", "-x", "Safari"], capture_output=True, text=True)
+ return [int(p) for p in r.stdout.split() if p.strip()]
+
+
+def _safari_window_count_via_driver(wait_s: float = 4.0) -> int:
+ """Spawn one cua-driver session and poll until Safari windows appear.
+
+ Waits up to ``wait_s`` seconds for Safari to create its window after
+ launch (Safari opens a URL asynchronously; the window may not be
+ registered in WindowServer immediately). Keeps one DriverClient open
+ for the duration to avoid spawning/killing multiple processes.
+ """
+ deadline = time.time() + wait_s
+ with DriverClient(default_binary_path()) as c:
+ while True:
+ r = c.call_tool("list_windows", {})
+ sc = r.get("structuredContent") or {}
+ wins = sc.get("windows") or []
+ count = sum(
+ 1 for w in wins if "safari" in w.get("app_name", "").lower()
+ )
+ if count > 0 or time.time() >= deadline:
+ return count
+ time.sleep(0.5)
+
+
+def _launch_focus_app() -> tuple[subprocess.Popen, int]:
+ proc = subprocess.Popen(
+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
+ )
+ for _ in range(40):
+ line = proc.stdout.readline().strip()
+ if line.startswith("FOCUS_PID="):
+ return proc, int(line.split("=", 1)[1])
+ time.sleep(0.1)
+ proc.terminate()
+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")
+
+
+# ---------------------------------------------------------------------------
+# Test
+# ---------------------------------------------------------------------------
+
+class TestHermesLaunchSafariBg(unittest.TestCase):
+ """Verify hermes launches Safari in the background without stealing focus."""
+
+ _focus_proc: subprocess.Popen
+ _focus_pid: int
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ if not _ANTHROPIC_KEY:
+ raise unittest.SkipTest(
+ "ANTHROPIC_API_KEY not set — skipping hermes Safari launch test"
+ )
+
+ # Build FocusMonitorApp if needed.
+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")
+ if (not os.path.exists(_FOCUS_APP_EXE)
+ or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)):
+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)
+
+ # Kill Safari so we can detect a fresh launch.
+ subprocess.run(["pkill", "-x", "Safari"], check=False)
+ time.sleep(1.5)
+ assert _safari_pids() == [], "Safari still running after pkill"
+
+ # Clear focus-loss counters.
+ for f in (_LOSS_FILE, _KEY_LOSS_FILE, _FIELD_LOSS_FILE):
+ if os.path.exists(f):
+ os.remove(f)
+
+ # FocusMonitorApp owns focus throughout.
+ cls._focus_proc, cls._focus_pid = _launch_focus_app()
+
+ # Stabilise: wait until FocusMonitorApp's field-loss counter stops changing.
+ prev = -1
+ for _ in range(20):
+ cur = _read_file_int(_FIELD_LOSS_FILE)
+ if cur == prev:
+ break
+ prev = cur
+ time.sleep(0.3)
+
+ cls._losses_before = _read_file_int(_LOSS_FILE)
+ cls._key_losses_before = _read_file_int(_KEY_LOSS_FILE)
+ cls._field_losses_before = _read_file_int(_FIELD_LOSS_FILE)
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ cls._focus_proc.terminate()
+ # Close stdout before waiting — if FocusMonitorApp keeps writing,
+ # the pipe buffer fills and terminate() blocks until it's drained.
+ if cls._focus_proc.stdout:
+ try:
+ cls._focus_proc.stdout.close()
+ except Exception:
+ pass
+ try:
+ cls._focus_proc.wait(timeout=3)
+ except subprocess.TimeoutExpired:
+ cls._focus_proc.kill()
+ cls._focus_proc.wait(timeout=2)
+ subprocess.run(["pkill", "-x", "Safari"], check=False)
+ # Kill any orphaned cua-driver processes left by hermes.
+ subprocess.run(["pkill", "-f", "cua-driver mcp"], check=False)
+
+ def _run_hermes(self, prompt: str, timeout: int = 240, max_turns: int = 8) -> str:
+ env = os.environ.copy()
+ env["ANTHROPIC_API_KEY"] = _ANTHROPIC_KEY
+ env["HERMES_COMPUTER_USE_BACKEND"] = "cua"
+ env["HERMES_CUA_DRIVER_CMD"] = default_binary_path()
+
+ result = subprocess.run(
+ [
+ _HERMES_BIN, "chat",
+ "--provider", "anthropic",
+ "-m", _MODEL,
+ "--toolsets", "computer_use",
+ "--yolo",
+ "-Q",
+ "--max-turns", str(max_turns),
+ "-q", prompt,
+ ],
+ env=env, capture_output=True, text=True, timeout=timeout,
+ )
+ return (result.stdout + "\n" + result.stderr).strip()
+
+ def _assert_no_new_focus_loss(self, allow_field_losses: int = 1) -> None:
+ time.sleep(0.3)
+ app_delta = _read_file_int(_LOSS_FILE) - self._losses_before
+ key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before
+ field_delta = _read_file_int(_FIELD_LOSS_FILE) - self._field_losses_before
+ msgs = []
+ if app_delta:
+ msgs.append(f"app lost focus {app_delta}x")
+ if key_delta:
+ msgs.append(f"window lost key status {key_delta}x")
+ if field_delta > allow_field_losses:
+ msgs.append(
+ f"text-input lost first-responder {field_delta}x "
+ f"(allowed {allow_field_losses})"
+ )
+ self.assertEqual(
+ len(msgs), 0,
+ "Focus stolen during launch_app: " + "; ".join(msgs),
+ )
+
+ def test_launch_safari_background_no_focus_steal(self) -> None:
+ """hermes launches Safari hidden with a URL; window exists; focus never stolen.
+
+ Safari needs a URL to create a window — launching bare (no URL) starts
+ the process but NSWorkspace never triggers window creation. The prompt
+ explicitly includes the URL so the model uses the right call.
+ """
+ out = self._run_hermes(
+ f"Use computer_use to launch Safari in the background. "
+ f"Call action='launch_app' with bundle_id='{_SAFARI_BUNDLE}' "
+ f"and urls=['{_LAUNCH_URL}']. "
+ f"After launching, call action='capture' with app='Safari' to verify "
+ f"Safari's window is reachable and report the window title or "
+ f"first AX element you see. "
+ f"Do NOT use action='focus_app' with raise_window=true.",
+ timeout=240,
+ max_turns=8,
+ )
+
+ # 1. Safari process must actually be running.
+ pids = _safari_pids()
+ self.assertTrue(
+ len(pids) > 0,
+ f"Safari process not found after hermes launch_app.\n"
+ f"hermes output (tail):\n{out[-800:]}",
+ )
+
+ # 2. cua-driver must see at least one Safari window.
+ # We ask the driver independently — this is the anti-hallucination check.
+ # Safari may need a moment to register its window in WindowServer after
+ # the URL loads, so _safari_window_count_via_driver polls for up to 4s.
+ window_count = _safari_window_count_via_driver(wait_s=4.0)
+ self.assertGreater(
+ window_count, 0,
+ f"cua-driver sees no Safari windows after launch_app+URL "
+ f"(got {window_count}). "
+ f"Safari may have launched without a window — ensure urls=['{_LAUNCH_URL}'] "
+ f"was passed.\nhermes output (tail):\n{out[-800:]}",
+ )
+
+ # 3. No focus stolen from FocusMonitorApp.
+ # allow_field_losses=1 for the known SyntheticAppFocusEnforcer AX
+ # side-effect that can briefly disturb field first-responder on the
+ # first AX interaction with a new pid, without stealing app focus.
+ self._assert_no_new_focus_loss(allow_field_losses=1)