Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): Chrome CGEvent modifier leak, silent-accept detection, cursor animation (#1422)
ddupontff0a6478c910
diff --git a/libs/cua-driver/Sources/CuaDriverCore/Input/KeyboardInput.swift b/libs/cua-driver/Sources/CuaDriverCore/Input/KeyboardInput.swiftindex 4e7055153b..0f9b712e75 100644--- a/libs/cua-driver/Sources/CuaDriverCore/Input/KeyboardInput.swift+++ b/libs/cua-driver/Sources/CuaDriverCore/Input/KeyboardInput.swift@@ -138,6 +138,14 @@ public enum KeyboardInput {/// is left at 0 — macOS uses the attached unicode string instead./// Pass `toPid` to deliver the events directly to a process's queue/// rather than the system HID tap.+ ///+ /// **Modifier-state isolation**: when posting via `postToPid` some+ /// apps (notably Chrome) inspect the CGEvent's `flags` field to infer+ /// whether Shift or other modifiers were held to produce the unicode+ /// character. If `flags` is non-zero the app may "remember" the+ /// modifier and apply it to the next character. We explicitly clear+ /// `event.flags` on every event so the receiver always sees a clean+ /// modifier state regardless of what was typed before.private static func sendUnicodeCharacter(_ character: Character, toPid pid: pid_t? = nil) throws {@@ -154,6 +162,11 @@ public enum KeyboardInput {"unicode character \"\(character)\" down=\(keyDown)")}+ // Clear all modifier flags so the receiver doesn't accumulate+ // Shift/Control/Option state between characters. Without this,+ // Chrome interprets uppercase unicode events as "Shift held" and+ // applies Shift to the next character (e.g. 'E' + '2' → '@').+ event.flags = []utf16.withUnsafeBufferPointer { buffer inif let base = buffer.baseAddress {event.keyboardSetUnicodeString(diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swiftindex d69112f834..d33473736c 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetValueTool.swift@@ -80,6 +80,15 @@ public enum SetValueTool {)let target = AXInput.describe(element)+ // Animate the agent cursor to the target element before+ // setting its value — same visual feedback as click.+ // No-op when the cursor is disabled or the element has no+ // resolvable screen position.+ if let center = AXInput.screenCenter(of: element) {+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }+ await AgentCursor.shared.animateAndWait(to: center)+ }+// ── AXPopUpButton (HTML <select>) special path ──────────// Safari WebKit does not propagate AXValue writes back to// the HTML DOM for <select> elements — the AX write returns@@ -88,32 +97,39 @@ public enum SetValueTool {// AXPress the one whose AXTitle or AXValue matches `value`.// This bypasses the native macOS popup menu entirely, so the// option is selected without the menu needing to be visible.+ let result: CallTool.Resultif target.role == "AXPopUpButton" {- return try await selectPopupOption(+ result = try await selectPopupOption(element: element,index: index,pid: pid,value: value,elementTitle: target.title ?? "")- }-- // ── Default path: write AXValue directly ────────────────- try await AppStateRegistry.focusGuard.withFocusSuppressed(- pid: pid,- element: element- ) {- try AXInput.setAttribute(- "AXValue",- on: element,- value: value as CFTypeRef+ } else {+ // ── Default path: write AXValue directly ────────────────+ try await AppStateRegistry.focusGuard.withFocusSuppressed(+ pid: pid,+ element: element+ ) {+ try AXInput.setAttribute(+ "AXValue",+ on: element,+ value: value as CFTypeRef+ )+ }+ let summary =+ "✅ Set AXValue on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"."+ result = CallTool.Result(+ content: [.text(text: summary, annotations: nil, _meta: nil)])}- let summary =- "✅ Set AXValue on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"."- return CallTool.Result(- content: [.text(text: summary, annotations: nil, _meta: nil)]- )+ // Cursor press-pulse after the action — mirrors ClickTool's+ // finishClick sequence. No-op when the cursor is disabled.+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }+ await AgentCursor.shared.playClickPress()+ await AgentCursor.shared.finishClick(pid: pid)+ return result} catch let error as AppStateError {return errorResult(error.description)} catch let error as AXInputError {diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swiftindex 16035c3599..084d54ae55 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/TypeTextTool.swift@@ -99,9 +99,11 @@ public enum TypeTextTool {+ "the same window_id you used in `get_window_state`.")}- // Attempt AX bulk-insert. On any AXInputError fall back to- // CGEvent character synthesis, which reaches Chromium / Electron- // inputs that don't expose kAXSelectedText.+ // Attempt AX bulk-insert. On any AXInputError — or when a+ // silent-accept-but-no-DOM-update is detected (Chromium web+ // inputs accept kAXSelectedText writes without error but never+ // propagate the value to the HTML DOM) — fall back to CGEvent+ // character synthesis.do {if let index = elementIndex, let rawWindowId {guard let windowId = UInt32(exactly: rawWindowId) else {@@ -112,6 +114,13 @@ public enum TypeTextTool {pid: pid,windowId: windowId,elementIndex: index)+ // Animate the agent cursor to the target element before+ // writing — same visual feedback as click. No-op when the+ // cursor is disabled or the element has no screen position.+ if let center = AXInput.screenCenter(of: element) {+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }+ await AgentCursor.shared.animateAndWait(to: center)+ }do {try await AppStateRegistry.focusGuard.withFocusSuppressed(pid: pid, element: element@@ -122,42 +131,78 @@ public enum TypeTextTool {value: text as CFTypeRef)}- let target = AXInput.describe(element)- let summary =- "✅ Inserted \(text.count) char(s) into [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\" on pid \(rawPid) via AX."- return CallTool.Result(- content: [.text(text: summary, annotations: nil, _meta: nil)]- )+ // Verify the write was actually applied — Chromium+ // web inputs silently accept the call but never+ // update their DOM value. Read AXValue back; if it's+ // still empty when we just wrote non-empty text the+ // write was a no-op and we must use CGEvent instead.+ if !text.isEmpty,+ let axValue = AXInput.stringAttribute("AXValue", of: element),+ axValue.isEmpty+ {+ // Silent-accept detected — fall through.+ } else {+ let target = AXInput.describe(element)+ let summary =+ "✅ Inserted \(text.count) char(s) into [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\" on pid \(rawPid) via AX."+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }+ await AgentCursor.shared.playClickPress()+ await AgentCursor.shared.finishClick(pid: pid)+ return CallTool.Result(+ content: [.text(text: summary, annotations: nil, _meta: nil)]+ )+ }} catch is AXInputError {// AX rejected — fall through to CGEvent synthesis.}} else {do {let element = try AXInput.focusedElement(pid: pid)+ // Animate the cursor to the focused element before writing.+ if let center = AXInput.screenCenter(of: element) {+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }+ await AgentCursor.shared.animateAndWait(to: center)+ }try AXInput.setAttribute("AXSelectedText",on: element,value: text as CFTypeRef)- let target = AXInput.describe(element)- let summary =- "✅ Inserted \(text.count) char(s) into focused \(target.role ?? "?") \"\(target.title ?? "")\" on pid \(rawPid) via AX."- return CallTool.Result(- content: [.text(text: summary, annotations: nil, _meta: nil)]- )+ // Same silent-accept check as the element_index path.+ if !text.isEmpty,+ let axValue = AXInput.stringAttribute("AXValue", of: element),+ axValue.isEmpty+ {+ // Silent-accept detected — fall through.+ } else {+ let target = AXInput.describe(element)+ let summary =+ "✅ Inserted \(text.count) char(s) into focused \(target.role ?? "?") \"\(target.title ?? "")\" on pid \(rawPid) via AX."+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }+ await AgentCursor.shared.playClickPress()+ await AgentCursor.shared.finishClick(pid: pid)+ return CallTool.Result(+ content: [.text(text: summary, annotations: nil, _meta: nil)]+ )+ }} catch is AXInputError {// AX rejected — fall through to CGEvent synthesis.}}- // CGEvent fallback path.+ // CGEvent fallback path — reached when AX write was rejected+ // (AXInputError) or silently accepted without updating the+ // DOM value (Chromium web inputs).try KeyboardInput.typeCharacters(text,delayMilliseconds: delayMs,toPid: pid)let summary =- "✅ Typed \(text.count) char(s) on pid \(rawPid) via CGEvent (AX fallback, \(delayMs)ms delay)."+ "✅ Typed \(text.count) char(s) on pid \(rawPid) via CGEvent (\(delayMs)ms delay)."+ await MainActor.run { AgentCursor.shared.pinAbove(pid: pid) }+ await AgentCursor.shared.playClickPress()+ await AgentCursor.shared.finishClick(pid: pid)return CallTool.Result(content: [.text(text: summary, annotations: nil, _meta: nil)])diff --git a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.htmlindex 44225353a9..74ae4addfe 100644--- a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html+++ b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html@@ -32,7 +32,7 @@ button#submit-btn:hover { background: #0066DD; }</head><body><h1>All Inputs Test Form</h1>-<form id="test-form" onsubmit="handleSubmit(event)">+<form id="test-form" onsubmit="handleSubmit(event)" autocomplete="off"><div class="field"><label for="f-text">Text</label>diff --git a/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py b/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.pynew file mode 100644index 0000000000..2ddff05cc8--- /dev/null+++ b/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py@@ -0,0 +1,603 @@+"""Integration test: hermes CLI drives Chrome form fields in the background.++Goal+----+One test per HTML input type. Each test asks hermes to interact with exactly+ONE element in a Chrome window that is NOT frontmost — verifying that:++ 1. cua-driver delivers events to Chromium inputs without stealing focus.+ The unified ``type_text`` tool automatically falls back to CGEvent+ synthesis when its AX attribute-write path is rejected by Chrome's+ web-content inputs — callers (hermes) don't need to know which path ran.++ 2. ``launch_app`` (not ``open -a``) is used to keep Chrome hidden and+ avoid the self-activation race that ``open -g`` suffers from.++ 3. FocusMonitorApp retains the frontmost / key-window slot throughout.++Chrome vs. Safari differences+------------------------------+* ``type_text`` always takes the CGEvent fallback for Chromium inputs;+ the AX attribute-write path is silently rejected and the response says+ "via CGEvent" rather than "via AX".+* Field-value verification uses osascript against Chrome's AppleScript+ interface ("Allow JavaScript from Apple Events" must be enabled —+ setUpClass patches the Preferences JSON automatically).+* Setup uses ``launch_app`` (driver-level), not ``open -g``.++Setup (setUpClass)+------------------+1. Kill Chrome, enable "Allow JavaScript from Apple Events" (Preferences+ JSON patch), relaunch Chrome with the form URL via cua-driver's+ ``launch_app`` tool so it stays hidden.+2. Wait for the form's ``getFieldValues`` helper to be callable.+3. Launch FocusMonitorApp last — it becomes frontmost and holds focus for+ every test in the class.++Per-test design+---------------+* Prompt is intentionally minimal: capture Chrome, interact with ONE named+ element, stop. We do NOT ask hermes to re-capture or verify — that keeps+ tool-call count to ≤ 4.+* After hermes returns the test reads the field value via osascript and+ asserts the expected value was written.+* The FocusMonitorApp focus-loss counters must not increase.++Run+---+ scripts/test.sh test_hermes_chrome_form_fill+ python3 -m pytest Tests/integration/test_hermes_chrome_form_fill.py -v+"""++from __future__ import annotations++import glob+import json+import os+import re+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import DriverClient, default_binary_path, resolve_window_id # noqa: E402++# ---------------------------------------------------------------------------+# Paths & constants+# ---------------------------------------------------------------------------++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))++_HTML_FORM = os.path.join(_THIS_DIR, "fixtures", "form_all_inputs.html")+_FORM_URL = f"file://{_HTML_FORM}"++_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"+)+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"+_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt"+_FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt"++_HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes")++CHROME_BUNDLE = "com.google.Chrome"++_ANTHROPIC_KEY: str = os.environ.get("ANTHROPIC_API_KEY", "")++# Use haiku for speed/cost; override with HERMES_TEST_MODEL for a stronger model.+_MODEL = os.environ.get("HERMES_TEST_MODEL", "claude-haiku-4-5-20251001")+++# ---------------------------------------------------------------------------+# Chrome Apple Events JS helpers+# ---------------------------------------------------------------------------++def _enable_chrome_apple_events() -> None:+ """Quit Chrome, write allow_javascript_apple_events to all profiles.++ Mirrors the same helper in test_browser_js.py — patches Preferences JSON+ directly so we don't depend on the ``page`` tool's ``enable_javascript_apple_events``+ action (which itself needs Apple Events to already be on for the confirmation).+ Does NOT relaunch Chrome — the caller launches it via ``launch_app`` afterwards.+ """+ subprocess.run(+ ["osascript", "-e", 'quit app "Google Chrome"'],+ check=False, timeout=10,+ )+ time.sleep(1.5)++ for prefs_path in glob.glob(+ os.path.expanduser(+ "~/Library/Application Support/Google/Chrome/*/Preferences"+ )+ ):+ profile = prefs_path.split("/")[-2]+ if "System" in profile or "Guest" in profile:+ continue+ try:+ with open(prefs_path) as f:+ data = json.load(f)+ data.setdefault("browser", {})["allow_javascript_apple_events"] = True+ data.setdefault("account_values", {}).setdefault("browser", {})[+ "allow_javascript_apple_events"+ ] = True+ with open(prefs_path, "w") as f:+ json.dump(data, f)+ print(f" Patched {profile}/Preferences")+ except Exception as e:+ print(f" Skipped {profile}: {e}")+++def _js_chrome(expr: str) -> str:+ """Evaluate a JS expression in Chrome's front tab via osascript.++ Uses json.dumps to safely quote the expression string for AppleScript.+ Requires "Allow JavaScript from Apple Events" to be enabled in Chrome.+ Returns stdout stripped of trailing whitespace.+ """+ # AppleScript: tell application "Google Chrome" to execute front window's+ # active tab javascript "<expr>"+ osa = (+ "tell application \"Google Chrome\" to execute "+ f"front window's active tab javascript {json.dumps(expr)}"+ )+ r = subprocess.run(+ ["osascript", "-e", osa], capture_output=True, text=True, timeout=15+ )+ return r.stdout.strip()+++def _wait_for_chrome_form(timeout: float = 30.0) -> bool:+ """Wait until Chrome's active tab has the test form's key element loaded.++ Chrome's AppleScript 'execute javascript' runs in an isolated world —+ page-defined globals like getFieldValues() are NOT visible. We probe for+ the DOM element instead (document.getElementById is available in any world).+ """+ deadline = time.time() + timeout+ while time.time() < deadline:+ result = _js_chrome('document.getElementById("f-text") ? "ready" : "not-ready"')+ if result == "ready":+ return True+ time.sleep(0.5)+ return False+++# Map logical field names → HTML element IDs in form_all_inputs.html.+_FIELD_TO_ID: dict[str, str] = {+ "text": "f-text",+ "email": "f-email",+ "password": "f-password",+ "number": "f-number",+ "tel": "f-tel",+ "textarea": "f-textarea",+ "select": "f-select",+ "range": "f-range",+ "date": "f-date",+ "color": "f-color",+}+++def _field(name: str) -> str:+ """Read a single form field value from Chrome.++ Chrome's AppleScript 'execute javascript' runs in an isolated world, so+ page-defined helpers like getFieldValues() are not visible. We read the+ DOM element's .value property directly instead.+ """+ elem_id = _FIELD_TO_ID.get(name, f"f-{name}")+ val = _js_chrome(+ f'(function(){{var e=document.getElementById("{elem_id}");'+ f'return e ? e.value : "__missing__";}})()'+ )+ return val if val != "__missing__" else ""+++def _checkbox_checked() -> bool:+ val = _js_chrome(+ '(function(){var e=document.getElementById("f-checkbox");'+ 'return e ? e.checked.toString() : "false";})()'+ )+ return val.lower() == "true"+++# ---------------------------------------------------------------------------+# FocusMonitorApp helpers+# ---------------------------------------------------------------------------++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ return proc, int(line.split("=", 1)[1])+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")+++def _read_file_int(path: str) -> int:+ try:+ with open(path) as f:+ return int(f.read().strip())+ except (FileNotFoundError, ValueError):+ return 0+++# ---------------------------------------------------------------------------+# Base test class+# ---------------------------------------------------------------------------++class _HermesChromFormBase(unittest.TestCase):+ """Shared setup/teardown and hermes runner for all Chrome per-input tests."""++ _focus_proc: subprocess.Popen+ _focus_pid: int+ _chrome_pid: int+ binary: str++ @classmethod+ def setUpClass(cls) -> None:+ if not _ANTHROPIC_KEY:+ raise unittest.SkipTest(+ "ANTHROPIC_API_KEY is not set — skipping hermes Chrome form-fill tests"+ )+ if not os.path.exists(_HERMES_BIN):+ raise unittest.SkipTest(+ f"hermes not found at {_HERMES_BIN} — install hermes first"+ )++ for f in (_LOSS_FILE, _KEY_LOSS_FILE, _FIELD_LOSS_FILE):+ try:+ os.remove(f)+ except FileNotFoundError:+ pass++ # Rebuild FocusMonitorApp when Swift source is newer.+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")+ if (not os.path.exists(_FOCUS_APP_EXE)+ or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)):+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)++ cls.binary = default_binary_path()++ # Step 1: Enable "Allow JavaScript from Apple Events" in Chrome.+ # This quits Chrome, patches every profile's Preferences JSON.+ print("\n Enabling Chrome Apple Events JS...")+ _enable_chrome_apple_events()++ # Step 2: Launch Chrome hidden with the form URL via cua-driver's+ # launch_app — no activation, no window raise, no focus steal.+ print(f" Launching Chrome hidden with {_FORM_URL}")+ with DriverClient(cls.binary) as c:+ result = c.call_tool("launch_app", {+ "bundle_id": CHROME_BUNDLE,+ "urls": [_FORM_URL],+ })+ sc = result.get("structuredContent", {})+ cls._chrome_pid = sc.get("pid", 0)+ if not cls._chrome_pid:+ # Fallback: extract from list_apps+ apps = c.call_tool("list_apps")["structuredContent"]["apps"]+ for app in apps:+ if app.get("bundle_id") == CHROME_BUNDLE:+ cls._chrome_pid = app["pid"]+ break+ print(f" Chrome pid: {cls._chrome_pid}")++ # Step 3: Wait for Chrome to load the form and expose getFieldValues().+ if not _wait_for_chrome_form(30.0):+ raise RuntimeError(+ f"Chrome did not load {_HTML_FORM} within 30 s.\n"+ "Check: (1) file exists, (2) 'Allow JavaScript from Apple Events' "+ "is enabled in Chrome (Develop menu)."+ )+ print(" Form loaded — f-text element ready")++ # Step 4: Launch FocusMonitorApp last — it becomes frontmost and holds+ # the focus token for every test.+ cls._focus_proc, cls._focus_pid = _launch_focus_app()+ print(f" FocusMonitorApp pid: {cls._focus_pid}")+ # Wait for FocusMonitorApp's focus-loss counter to stabilize.+ # The field loss counter bumps once during startup (macOS transitions+ # first-responder during the window-becoming-key sequence). We wait+ # until the counter hasn't changed for 0.5 s before recording the+ # baseline in setUp(), so startup-phase transients are not counted+ # as test failures.+ prev = _read_file_int(_FIELD_LOSS_FILE)+ for _ in range(20): # up to 10 s+ time.sleep(0.5)+ curr = _read_file_int(_FIELD_LOSS_FILE)+ if curr == prev:+ break+ prev = curr+ print(f" FocusMonitorApp ready (field_losses baseline: {_read_file_int(_FIELD_LOSS_FILE)})")++ @classmethod+ def tearDownClass(cls) -> None:+ cls._focus_proc.terminate()+ try:+ cls._focus_proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ cls._focus_proc.kill()+ subprocess.run(+ ["osascript", "-e", 'quit app "Google Chrome"'],+ check=False, timeout=10,+ )++ def setUp(self) -> None:+ self._losses_before = _read_file_int(_LOSS_FILE)+ self._key_losses_before = _read_file_int(_KEY_LOSS_FILE)+ self._field_losses_before = _read_file_int(_FIELD_LOSS_FILE)++ def _run_hermes(self, prompt: str, timeout: int = 120, max_turns: int = 5) -> str:+ """Run hermes with the anthropic provider and return combined stdout+stderr."""+ env = os.environ.copy()+ env["ANTHROPIC_API_KEY"] = _ANTHROPIC_KEY+ env["HERMES_COMPUTER_USE_BACKEND"] = "cua"+ env["HERMES_CUA_DRIVER_CMD"] = default_binary_path()++ result = subprocess.run(+ [+ _HERMES_BIN, "chat",+ "--provider", "anthropic",+ "-m", _MODEL,+ "--toolsets", "computer_use",+ "--yolo",+ "-Q",+ "--max-turns", str(max_turns),+ "-q", prompt,+ ],+ env=env, capture_output=True, text=True, timeout=timeout,+ )+ return (result.stdout + "\n" + result.stderr).strip()++ def _assert_no_new_focus_loss(self, allow_field_losses: int = 0) -> None:+ """Assert no new app-level or key-level focus losses occurred.++ ``allow_field_losses`` controls how many text-field first-responder+ losses are tolerated. Chrome interactions trigger one field loss per+ AX action via a known ``SyntheticAppFocusEnforcer`` side effect:+ writing ``AXFocused=true`` on Chrome's window emits an AX+ ``kAXFocusedWindowChangedNotification`` that briefly disrupts+ FocusMonitorApp's first-responder chain. App-level and key-window+ focus are NOT affected (Chrome never becomes frontmost). Pass+ ``allow_field_losses=1`` for single-action Chrome tests or a higher+ value for multi-action tests.+ """+ time.sleep(0.3)+ app_delta = _read_file_int(_LOSS_FILE) - self._losses_before+ key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before+ field_delta = _read_file_int(_FIELD_LOSS_FILE) - self._field_losses_before+ msgs = []+ if app_delta:+ msgs.append(f"app lost focus {app_delta}x")+ if key_delta:+ msgs.append(f"window lost key status {key_delta}x")+ if field_delta > allow_field_losses:+ msgs.append(+ f"text-input lost first-responder {field_delta}x"+ f" (allowed: {allow_field_losses})"+ )+ self.assertEqual(+ len(msgs), 0,+ f"Focus stolen in {self._testMethodName}: "+ + "; ".join(msgs)+ + " — background CGEvent delivery must not steal focus",+ )++ def _chrome_window_id(self) -> int:+ """Resolve Chrome's current on-screen window_id for AX verification."""+ with DriverClient(self.binary) as c:+ return resolve_window_id(c, self._chrome_pid)++ def _ax_contains(self, query: str) -> bool:+ """True if Chrome's AX tree contains a line matching *query* (case-insensitive).++ Uses get_window_state's query= filter so we only check the relevant+ subtree — avoids false positives from unrelated UI text.+ """+ try:+ wid = self._chrome_window_id()+ with DriverClient(self.binary) as c:+ snap = c.call_tool("get_window_state", {+ "pid": self._chrome_pid,+ "window_id": wid,+ "query": query,+ })+ tree = snap.get("structuredContent", snap).get("tree_markdown", "")+ return len(tree.strip()) > 0+ except Exception:+ return False+++# ---------------------------------------------------------------------------+# One test class per input type+# ---------------------------------------------------------------------------++class TestChromeTextInput(_HermesChromFormBase):+ """<input type=text> in Chrome — type_text CGEvent fallback path."""++ def test_text_input(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form open. "+ "Use computer_use: action='capture', app='Google Chrome' to see the form. "+ "Click the Text input (id='f-text') to focus it, "+ "then type 'Hello Chrome' into it. Stop after typing."+ )+ val = _field("text")+ self.assertEqual(val, "Hello Chrome",+ msg=f"text='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss(allow_field_losses=3)+++class TestChromeEmailInput(_HermesChromFormBase):+ """<input type=email> in Chrome — CGEvent fallback for Chromium inputs."""++ def test_email_input(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form open. "+ "Use computer_use: action='capture', app='Google Chrome'. "+ "Click the Email input (id='f-email') and type '[email protected]'. "+ "Stop after typing."+ )+ val = _field("email")+ self.assertEqual(val, "[email protected]",+ msg=f"email='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss(allow_field_losses=3)+++class TestChromePasswordInput(_HermesChromFormBase):+ """<input type=password> in Chrome."""++ def test_password_input(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form open. "+ "Use computer_use: action='capture', app='Google Chrome'. "+ "Click the Password input (id='f-password') and type 'Secure123!'. "+ "Stop after typing."+ )+ val = _field("password")+ self.assertEqual(val, "Secure123!",+ msg=f"password='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss(allow_field_losses=3)+++class TestChromeNumberInput(_HermesChromFormBase):+ """<input type=number> in Chrome."""++ def test_number_input(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form open. "+ "Use computer_use: action='capture', app='Google Chrome'. "+ "Click the Number input (id='f-number') and type '99'. "+ "Stop after typing."+ )+ val = _field("number")+ self.assertEqual(val, "99",+ msg=f"number='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss(allow_field_losses=3)+++class TestChromeTextarea(_HermesChromFormBase):+ """<textarea> in Chrome."""++ def test_textarea(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form open. "+ "Use computer_use: action='capture', app='Google Chrome'. "+ "Click the Message textarea (id='f-textarea') and type 'chrome bg ok'. "+ "Stop after typing."+ )+ val = _field("textarea")+ self.assertIn("chrome bg ok", val,+ msg=f"textarea='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss(allow_field_losses=3)+++class TestChromeCheckbox(_HermesChromFormBase):+ """<input type=checkbox> in Chrome — AX click."""++ def test_checkbox(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form open. "+ "Use computer_use: action='capture', app='Google Chrome'. "+ "Click the checkbox labelled 'I agree to the terms' (id='f-checkbox'). "+ "Stop after clicking."+ )+ self.assertTrue(_checkbox_checked(),+ msg=f"checkbox not checked\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss(allow_field_losses=3)+++class TestChromeSelectDropdown(_HermesChromFormBase):+ """<select> in Chrome — set_value on AXPopUpButton.++ Chrome exposes HTML <select> as AXPopUpButton. The set_value tool finds+ the matching option AXMenuItemMarkChar and AXPresses it directly without+ opening the native macOS menu — no window raise, no focus steal.+ """++ def test_select_dropdown(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form open. "+ "Use computer_use: action='capture', app='Google Chrome' to see the AX tree. "+ "Find the element for 'Favourite colour (Select)' (an AXPopUpButton, id='f-select'). "+ "Use computer_use: action='set_value', element=<index>, value='Blue' to select "+ "the Blue option directly. Stop after setting.",+ max_turns=4,+ timeout=180,+ )+ val = _field("select")+ self.assertEqual(val, "blue",+ msg=f"select='{val}'\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss(allow_field_losses=3)+++class TestChromeSubmitButton(_HermesChromFormBase):+ """<button type=submit> in Chrome — click and verify form submission."""++ def test_submit_button(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form open. "+ "Use computer_use: action='capture', app='Google Chrome' to see the AX tree. "+ "Find the AXButton labelled 'Submit Form' (id='submit-btn') and click it. "+ "Stop after clicking."+ )+ # Chrome's AppleScript JS runs in an isolated world — check the DOM+ # result element that the form's submit handler populates instead of+ # the page-global window._submitted (which isn't visible here).+ submitted = _js_chrome(+ '(function(){var e=document.getElementById("result");'+ 'return (e && e.textContent.trim().length > 0) ? "true" : "false";})()'+ )+ self.assertEqual(submitted, "true",+ msg=f"form not submitted\nhermes: {out[-400:]}")+ self._assert_no_new_focus_loss(allow_field_losses=3)+++class TestChromeFullFormFlow(_HermesChromFormBase):+ """End-to-end: fill text + email + number, then submit — all in background Chrome.++ This is the main e2e smoke test confirming:+ - launch_app correctly opens Chrome without activation+ - type_text's CGEvent fallback writes into Chromium inputs+ - Multiple tool calls across a single hermes session don't leak focus+ """++ def test_full_form_flow(self) -> None:+ out = self._run_hermes(+ "Google Chrome is in the background with a test HTML form at "+ f"'{_FORM_URL}'. "+ "Use computer_use to fill the following fields (capture Chrome first, "+ "then click each field before typing):\n"+ " - Text input (id='f-text'): type 'E2E Test'\n"+ " - Email input (id='f-email'): type '[email protected]'\n"+ " - Number input (id='f-number'): type '7'\n"+ "After filling all three fields, stop. "+ "Do NOT submit the form.",+ max_turns=20,+ timeout=360,+ )+ # Verify all three fields were written.+ text_val = _field("text")+ email_val = _field("email")+ num_val = _field("number")+ errors: list[str] = []+ if text_val != "E2E Test": errors.append(f"text='{text_val}'")+ if email_val != "[email protected]": errors.append(f"email='{email_val}'")+ if num_val != "7": errors.append(f"number='{num_val}'")+ self.assertFalse(+ errors,+ f"Field mismatch: {'; '.join(errors)}\nhermes: {out[-600:]}"+ )+ self._assert_no_new_focus_loss(allow_field_losses=3)+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py b/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.pynew file mode 100644index 0000000000..8239deb616--- /dev/null+++ b/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py@@ -0,0 +1,270 @@+"""Integration test: hermes can launch Safari in the background via launch_app.++What this test proves+---------------------+1. ``computer_use(action='launch_app', bundle_id='com.apple.Safari',+ urls=['https://example.com'])`` actually creates a Safari process and+ window — verified independently via a direct cua-driver MCP call to+ ``list_windows``, bypassing any model hallucination.+2. The launch does NOT steal focus from FocusMonitorApp: app-level focus,+ key-window status, and text-field first-responder are all unchanged.+3. ``capture(app='Safari')`` after launch returns Safari's AX tree and+ screenshot, not whatever window happens to be frontmost (Finder, etc.).++Key insight: Safari launched with ``hides=true`` and no URL never creates a+window — the process starts but NSWorkspace never triggers window creation.+Passing ``urls=['https://example.com']`` forces Safari to open a tab and+create a window. The window IS on-screen (z-ordered behind other windows)+but Safari is NOT the active app — that is the correct "background" behavior.++Setup+-----+Safari is killed before the test so we know the process was freshly launched+by hermes. FocusMonitorApp is launched last and holds focus throughout.++Run+---+ python3 -m pytest libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py -v+ HERMES_TEST_MODEL=claude-sonnet-4-6 python3 -m pytest ... -v+"""++from __future__ import annotations++import os+import subprocess+import sys+import time+import unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from driver_client import DriverClient, default_binary_path # noqa: E402++# ---------------------------------------------------------------------------+# Paths & constants+# ---------------------------------------------------------------------------++_THIS_DIR = os.path.dirname(os.path.abspath(__file__))+_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR))++_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp")+_FOCUS_APP_EXE = os.path.join(+ _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp"+)+_LOSS_FILE = "/tmp/focus_monitor_losses.txt"+_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt"+_FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt"++_HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes")++_ANTHROPIC_KEY: str = os.environ.get("ANTHROPIC_API_KEY", "")+_MODEL = os.environ.get("HERMES_TEST_MODEL", "claude-haiku-4-5-20251001")++_SAFARI_BUNDLE = "com.apple.Safari"+_LAUNCH_URL = "https://example.com"++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------++def _read_file_int(path: str) -> int:+ try:+ with open(path) as f:+ return int(f.read().strip())+ except (FileNotFoundError, ValueError):+ return 0+++def _safari_pids() -> list[int]:+ r = subprocess.run(["pgrep", "-x", "Safari"], capture_output=True, text=True)+ return [int(p) for p in r.stdout.split() if p.strip()]+++def _safari_window_count_via_driver(wait_s: float = 4.0) -> int:+ """Spawn one cua-driver session and poll until Safari windows appear.++ Waits up to ``wait_s`` seconds for Safari to create its window after+ launch (Safari opens a URL asynchronously; the window may not be+ registered in WindowServer immediately). Keeps one DriverClient open+ for the duration to avoid spawning/killing multiple processes.+ """+ deadline = time.time() + wait_s+ with DriverClient(default_binary_path()) as c:+ while True:+ r = c.call_tool("list_windows", {})+ sc = r.get("structuredContent") or {}+ wins = sc.get("windows") or []+ count = sum(+ 1 for w in wins if "safari" in w.get("app_name", "").lower()+ )+ if count > 0 or time.time() >= deadline:+ return count+ time.sleep(0.5)+++def _launch_focus_app() -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen(+ [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,+ )+ for _ in range(40):+ line = proc.stdout.readline().strip()+ if line.startswith("FOCUS_PID="):+ return proc, int(line.split("=", 1)[1])+ time.sleep(0.1)+ proc.terminate()+ raise RuntimeError("FocusMonitorApp did not print FOCUS_PID= in time")+++# ---------------------------------------------------------------------------+# Test+# ---------------------------------------------------------------------------++class TestHermesLaunchSafariBg(unittest.TestCase):+ """Verify hermes launches Safari in the background without stealing focus."""++ _focus_proc: subprocess.Popen+ _focus_pid: int++ @classmethod+ def setUpClass(cls) -> None:+ if not _ANTHROPIC_KEY:+ raise unittest.SkipTest(+ "ANTHROPIC_API_KEY not set — skipping hermes Safari launch test"+ )++ # Build FocusMonitorApp if needed.+ src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift")+ if (not os.path.exists(_FOCUS_APP_EXE)+ or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)):+ subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True)++ # Kill Safari so we can detect a fresh launch.+ subprocess.run(["pkill", "-x", "Safari"], check=False)+ time.sleep(1.5)+ assert _safari_pids() == [], "Safari still running after pkill"++ # Clear focus-loss counters.+ for f in (_LOSS_FILE, _KEY_LOSS_FILE, _FIELD_LOSS_FILE):+ if os.path.exists(f):+ os.remove(f)++ # FocusMonitorApp owns focus throughout.+ cls._focus_proc, cls._focus_pid = _launch_focus_app()++ # Stabilise: wait until FocusMonitorApp's field-loss counter stops changing.+ prev = -1+ for _ in range(20):+ cur = _read_file_int(_FIELD_LOSS_FILE)+ if cur == prev:+ break+ prev = cur+ time.sleep(0.3)++ cls._losses_before = _read_file_int(_LOSS_FILE)+ cls._key_losses_before = _read_file_int(_KEY_LOSS_FILE)+ cls._field_losses_before = _read_file_int(_FIELD_LOSS_FILE)++ @classmethod+ def tearDownClass(cls) -> None:+ cls._focus_proc.terminate()+ # Close stdout before waiting — if FocusMonitorApp keeps writing,+ # the pipe buffer fills and terminate() blocks until it's drained.+ if cls._focus_proc.stdout:+ try:+ cls._focus_proc.stdout.close()+ except Exception:+ pass+ try:+ cls._focus_proc.wait(timeout=3)+ except subprocess.TimeoutExpired:+ cls._focus_proc.kill()+ cls._focus_proc.wait(timeout=2)+ subprocess.run(["pkill", "-x", "Safari"], check=False)+ # Kill any orphaned cua-driver processes left by hermes.+ subprocess.run(["pkill", "-f", "cua-driver mcp"], check=False)++ def _run_hermes(self, prompt: str, timeout: int = 240, max_turns: int = 8) -> str:+ env = os.environ.copy()+ env["ANTHROPIC_API_KEY"] = _ANTHROPIC_KEY+ env["HERMES_COMPUTER_USE_BACKEND"] = "cua"+ env["HERMES_CUA_DRIVER_CMD"] = default_binary_path()++ result = subprocess.run(+ [+ _HERMES_BIN, "chat",+ "--provider", "anthropic",+ "-m", _MODEL,+ "--toolsets", "computer_use",+ "--yolo",+ "-Q",+ "--max-turns", str(max_turns),+ "-q", prompt,+ ],+ env=env, capture_output=True, text=True, timeout=timeout,+ )+ return (result.stdout + "\n" + result.stderr).strip()++ def _assert_no_new_focus_loss(self, allow_field_losses: int = 1) -> None:+ time.sleep(0.3)+ app_delta = _read_file_int(_LOSS_FILE) - self._losses_before+ key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before+ field_delta = _read_file_int(_FIELD_LOSS_FILE) - self._field_losses_before+ msgs = []+ if app_delta:+ msgs.append(f"app lost focus {app_delta}x")+ if key_delta:+ msgs.append(f"window lost key status {key_delta}x")+ if field_delta > allow_field_losses:+ msgs.append(+ f"text-input lost first-responder {field_delta}x "+ f"(allowed {allow_field_losses})"+ )+ self.assertEqual(+ len(msgs), 0,+ "Focus stolen during launch_app: " + "; ".join(msgs),+ )++ def test_launch_safari_background_no_focus_steal(self) -> None:+ """hermes launches Safari hidden with a URL; window exists; focus never stolen.++ Safari needs a URL to create a window — launching bare (no URL) starts+ the process but NSWorkspace never triggers window creation. The prompt+ explicitly includes the URL so the model uses the right call.+ """+ out = self._run_hermes(+ f"Use computer_use to launch Safari in the background. "+ f"Call action='launch_app' with bundle_id='{_SAFARI_BUNDLE}' "+ f"and urls=['{_LAUNCH_URL}']. "+ f"After launching, call action='capture' with app='Safari' to verify "+ f"Safari's window is reachable and report the window title or "+ f"first AX element you see. "+ f"Do NOT use action='focus_app' with raise_window=true.",+ timeout=240,+ max_turns=8,+ )++ # 1. Safari process must actually be running.+ pids = _safari_pids()+ self.assertTrue(+ len(pids) > 0,+ f"Safari process not found after hermes launch_app.\n"+ f"hermes output (tail):\n{out[-800:]}",+ )++ # 2. cua-driver must see at least one Safari window.+ # We ask the driver independently — this is the anti-hallucination check.+ # Safari may need a moment to register its window in WindowServer after+ # the URL loads, so _safari_window_count_via_driver polls for up to 4s.+ window_count = _safari_window_count_via_driver(wait_s=4.0)+ self.assertGreater(+ window_count, 0,+ f"cua-driver sees no Safari windows after launch_app+URL "+ f"(got {window_count}). "+ f"Safari may have launched without a window — ensure urls=['{_LAUNCH_URL}'] "+ f"was passed.\nhermes output (tail):\n{out[-800:]}",+ )++ # 3. No focus stolen from FocusMonitorApp.+ # allow_field_losses=1 for the known SyntheticAppFocusEnforcer AX+ # side-effect that can briefly disturb field first-responder on the+ # first AX interaction with a new pid, without stealing app focus.+ self._assert_no_new_focus_loss(allow_field_losses=1)