Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): verify exact macOS window activation (#2829)

Francesco Bonaccicd5c6f3ec5c2
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
index c862c5ac5c..49bd4529f6 100644
--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx
+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
@@ -189,7 +189,7 @@ Force-terminate a process by pid (kill -9 equivalent on macOS / Linux; taskkill
### `bring_to_front`
-Persistently activate an app so it genuinely holds macOS foreground, then leave it there. Most input does NOT need this — every macOS dispatch reaches backgrounded windows, and `delivery_mode:"foreground"` does its own brief front→act→restore. Reach for `bring_to_front` only for a focus-proxy surface that re-arms its own input channel on activation and must stay frontmost across the interaction — chiefly a remote-desktop client (Microsoft Windows App / RDP), where the brief flash drops keystrokes. Fronts the exact macOS window through WindowServer when `window_id` is present, with app-level Cocoa activation as a fallback. This DOES steal foreground — explicit opt-in, never used by the input ladder.
+Persistently activate an app and leave it in the foreground. Most input does not need this; use it only for a focus-proxy surface that must remain foreground across interactions. With window_id, success means the exact ordinary macOS window was independently verified as the focused window and first in WindowServer layer-0 order. Request acceptance alone is reported as a partial result, never as activation. This DOES steal foreground.
**Arguments:**
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock
index 9ccddd4a1e..d1fada2512 100644
--- a/libs/cua-driver/rust/Cargo.lock
+++ b/libs/cua-driver/rust/Cargo.lock
@@ -986,6 +986,7 @@ dependencies = [
"anyhow",
"async-trait",
"base64",
+ "core-foundation",
"cua-driver-contract",
"cua-driver-core",
"cua-driver-sdk",
diff --git a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
index 3fa4c61fe6..02435abd18 100644
--- a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
+++ b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
@@ -97,3 +97,6 @@ windows = { version = "0.61", features = [
"Win32_Security",
"Win32_System_Threading",
] }
+
+[target.'cfg(target_os = "macos")'.dev-dependencies]
+core-foundation = "0.10"
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/bring_to_front_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/bring_to_front_macos_test.rs
new file mode 100644
index 0000000000..48ede776b6
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/bring_to_front_macos_test.rs
@@ -0,0 +1,421 @@
+//! Exact-window certification for macOS `bring_to_front`.
+//!
+//! The assertions deliberately do not consume cua-driver's `list_windows` or
+//! internal verifier. System Events independently reports the frontmost process
+//! and AX focused window, while a direct CoreGraphics query provides the
+//! WindowServer front-to-back order.
+//!
+//! Run with:
+//! `cargo test -p cua-driver --test bring_to_front_macos_test -- --ignored --nocapture --test-threads=1`
+
+#![cfg(target_os = "macos")]
+
+use std::collections::{HashMap, HashSet};
+use std::ffi::c_void;
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+use core_foundation::{
+ array::{CFArray, CFArrayRef},
+ base::{CFGetTypeID, CFTypeRef, TCFType},
+ boolean::CFBoolean,
+ dictionary::CFDictionary,
+ number::CFNumber,
+ string::CFString,
+};
+use cua_driver_testkit::{harness_app, Driver, McpDriver};
+
+const MAIN_TITLE: &str = "CuaTestHarness AppKit";
+const SECONDARY_TITLE: &str = "CuaTestHarness AppKit Secondary";
+const OCCLUDER_TITLE: &str = "CuaTestHarness SwiftUI";
+#[link(name = "CoreGraphics", kind = "framework")]
+extern "C" {
+ fn CGWindowListCopyWindowInfo(option: u32, relative_to_window: u32) -> CFArrayRef;
+}
+
+#[derive(Debug, Clone)]
+struct OracleWindow {
+ id: u32,
+ pid: u32,
+ layer: i32,
+}
+
+struct Fixture {
+ pid: u32,
+ report: tempfile::NamedTempFile,
+}
+
+struct Occluder {
+ pid: u32,
+}
+
+impl Drop for Occluder {
+ fn drop(&mut self) {
+ let _ = Command::new("kill")
+ .args(["-TERM", &self.pid.to_string()])
+ .status();
+ }
+}
+
+fn harness_exe() -> std::path::PathBuf {
+ std::env::var("HARNESS_APPKIT_APP")
+ .map(std::path::PathBuf::from)
+ .ok()
+ .filter(|path| path.exists())
+ .unwrap_or_else(|| harness_app("harness-appkit", "CuaTestHarness.AppKit.app"))
+ .join("Contents/MacOS/CuaTestHarness.AppKit")
+}
+
+fn occluder_exe() -> std::path::PathBuf {
+ std::env::var("HARNESS_SWIFTUI_APP")
+ .map(std::path::PathBuf::from)
+ .ok()
+ .filter(|path| path.exists())
+ .unwrap_or_else(|| harness_app("harness-swiftui", "CuaTestHarness.SwiftUI.app"))
+ .join("Contents/MacOS/CuaTestHarness.SwiftUI")
+}
+
+fn launch(driver: &mut McpDriver, mode: Option<&str>) -> Fixture {
+ let exe = harness_exe();
+ assert!(exe.exists(), "required AppKit harness missing at {exe:?}");
+ let report = tempfile::NamedTempFile::new().expect("create AppKit window report");
+ let mut command = Command::new(exe);
+ command.stdout(Stdio::null()).stderr(Stdio::null());
+ command.env("CUA_HARNESS_WINDOW_REPORT", report.path());
+ if let Some(mode) = mode {
+ command.env("CUA_HARNESS_BRING_TO_FRONT_MODE", mode);
+ }
+ let child = cua_driver_testkit::spawn_in_job(&mut command).expect("launch AppKit harness");
+ let pid = child.id();
+ driver.reaper().push(child);
+ Fixture { pid, report }
+}
+
+fn running_executable_pids(exe: &std::path::Path) -> HashSet<u32> {
+ let output = Command::new("pgrep")
+ .args([
+ "-f",
+ "-x",
+ "--",
+ exe.to_str().expect("UTF-8 SwiftUI fixture path"),
+ ])
+ .output()
+ .expect("query SwiftUI fixture processes");
+ if !output.status.success() {
+ return HashSet::new();
+ }
+ String::from_utf8_lossy(&output.stdout)
+ .lines()
+ .filter_map(|line| line.trim().parse().ok())
+ .collect()
+}
+
+fn launch_occluder(_driver: &mut McpDriver) -> Occluder {
+ let exe = occluder_exe();
+ assert!(
+ exe.exists(),
+ "required distinct-bundle SwiftUI occluder missing at {exe:?}"
+ );
+ let app = exe
+ .ancestors()
+ .nth(3)
+ .expect("SwiftUI executable is inside an app bundle");
+ let previous = running_executable_pids(&exe);
+ let opened = Command::new("open")
+ .args(["-n", "-g"])
+ .arg(app)
+ .status()
+ .expect("launch SwiftUI occluder through LaunchServices");
+ assert!(opened.success(), "LaunchServices rejected SwiftUI occluder");
+
+ let deadline = Instant::now() + Duration::from_secs(12);
+ loop {
+ if let Some(pid) = running_executable_pids(&exe).difference(&previous).next() {
+ return Occluder { pid: *pid };
+ }
+ assert!(
+ Instant::now() < deadline,
+ "LaunchServices SwiftUI process did not appear"
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+}
+
+fn wait_for_ordinary_window(pid: u32) -> u32 {
+ let deadline = Instant::now() + Duration::from_secs(12);
+ loop {
+ if let Some(window) = cg_windows()
+ .into_iter()
+ .find(|window| window.pid == pid && window.layer == 0)
+ {
+ return window.id;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "ordinary window for pid {pid} did not appear"
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+}
+
+fn wait_for_window_ids(fixture: &Fixture, names: &[&str]) -> HashMap<String, u32> {
+ let deadline = Instant::now() + Duration::from_secs(12);
+ loop {
+ let report = std::fs::read_to_string(fixture.report.path()).unwrap_or_default();
+ let ids: HashMap<String, u32> = report
+ .lines()
+ .filter_map(|line| line.split_once('='))
+ .filter_map(|(name, id)| id.parse().ok().map(|id| (name.to_owned(), id)))
+ .collect();
+ let windows = cg_windows();
+ if names.iter().all(|name| {
+ ids.get(*name).is_some_and(|id| {
+ windows
+ .iter()
+ .any(|window| window.pid == fixture.pid && window.id == *id)
+ })
+ }) {
+ return ids;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "windows {names:?} for pid {} did not appear; report={report:?}; observed={windows:?}",
+ fixture.pid
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+}
+
+fn cg_windows() -> Vec<OracleWindow> {
+ // On-screen only | exclude desktop elements. CoreGraphics returns front to back.
+ let raw = unsafe { CGWindowListCopyWindowInfo(1 | 16, 0) };
+ assert!(!raw.is_null(), "CGWindowListCopyWindowInfo returned null");
+ let array: CFArray<CFTypeRef> = unsafe { CFArray::wrap_under_create_rule(raw) };
+ let mut windows = Vec::new();
+ for item in array.iter() {
+ let item = *item;
+ if unsafe { CFGetTypeID(item) } != CFDictionary::<*const c_void, *const c_void>::type_id() {
+ continue;
+ }
+ let dictionary: CFDictionary<*const c_void, *const c_void> =
+ unsafe { CFDictionary::wrap_under_get_rule(item as _) };
+ let number = |key: &str| -> i64 {
+ let key = CFString::new(key);
+ dictionary
+ .find(key.as_concrete_TypeRef() as *const c_void)
+ .and_then(|value| unsafe {
+ let value = *value;
+ (CFGetTypeID(value) == CFNumber::type_id())
+ .then(|| CFNumber::wrap_under_get_rule(value as _).to_i64())
+ .flatten()
+ })
+ .unwrap_or_default()
+ };
+ let on_screen = {
+ let key = CFString::new("kCGWindowIsOnscreen");
+ dictionary
+ .find(key.as_concrete_TypeRef() as *const c_void)
+ .is_some_and(|value| unsafe {
+ let value = *value;
+ CFGetTypeID(value) == CFBoolean::type_id()
+ && bool::from(CFBoolean::wrap_under_get_rule(value as _))
+ })
+ };
+ if on_screen {
+ windows.push(OracleWindow {
+ id: number("kCGWindowNumber") as u32,
+ pid: number("kCGWindowOwnerPID") as u32,
+ layer: number("kCGWindowLayer") as i32,
+ });
+ }
+ }
+ windows
+}
+
+fn frontmost_pid() -> u32 {
+ let output = Command::new("osascript")
+ .args([
+ "-e",
+ "tell application \"System Events\" to unix id of first process whose frontmost is true",
+ ])
+ .output()
+ .expect("query frontmost process");
+ assert!(
+ output.status.success(),
+ "frontmost oracle failed: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ String::from_utf8_lossy(&output.stdout)
+ .trim()
+ .parse()
+ .expect("frontmost oracle pid")
+}
+
+fn focused_window_title(pid: u32) -> String {
+ let script = format!(
+ "tell application \"System Events\" to tell (first process whose unix id is {pid}) to get name of value of attribute \"AXFocusedWindow\""
+ );
+ let output = Command::new("osascript")
+ .args(["-e", &script])
+ .output()
+ .expect("query AX focused window");
+ assert!(
+ output.status.success(),
+ "focused-window oracle failed: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ String::from_utf8_lossy(&output.stdout).trim().to_owned()
+}
+
+fn activate_and_raise(pid: u32, title: &str) {
+ let script = format!(
+ "tell application \"System Events\" to tell (first process whose unix id is {pid})\nset frontmost to true\nperform action \"AXRaise\" of window \"{title}\"\nend tell"
+ );
+ let output = Command::new("osascript")
+ .args(["-e", &script])
+ .output()
+ .expect("activate fixture window");
+ assert!(
+ output.status.success(),
+ "fixture activation failed: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ std::thread::sleep(Duration::from_millis(250));
+}
+
+fn layer_zero_front() -> OracleWindow {
+ cg_windows()
+ .into_iter()
+ .find(|window| window.layer == 0)
+ .expect("at least one ordinary on-screen window")
+}
+
+#[test]
+#[ignore]
+fn exact_secondary_window_is_verified_and_prior_app_can_recover_focus() {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named("macos-bring-to-front-exact")
+ .expect("start installed macOS daemon proxy");
+ let target = launch(&mut driver, Some("two-windows"));
+ let target_windows = wait_for_window_ids(&target, &["main", "secondary"]);
+ let secondary = target_windows["secondary"];
+ // Use a distinct bundle identity for the prior app. Raw-launching a second
+ // copy of the AppKit fixture lets AX and WindowServer select that process's
+ // window while NSWorkspace continues to report the other same-bundle
+ // instance as frontmost, which is not representative app recovery.
+ let occluder = launch_occluder(&mut driver);
+ let occluder_main = wait_for_ordinary_window(occluder.pid);
+
+ activate_and_raise(target.pid, MAIN_TITLE);
+ activate_and_raise(occluder.pid, OCCLUDER_TITLE);
+ assert_ne!(
+ layer_zero_front().id,
+ secondary,
+ "precondition: secondary already frontmost"
+ );
+
+ let result = driver.call(
+ "bring_to_front",
+ serde_json::json!({"pid": target.pid, "window_id": secondary}),
+ );
+ assert!(
+ !result.is_error(),
+ "exact bring_to_front failed: {}; raw={}",
+ result.text(),
+ result.raw
+ );
+ assert_eq!(result.structured()["activated"], true);
+ assert_eq!(result.structured()["request_accepted"], true);
+ assert_eq!(result.structured()["process_activated"], true);
+ assert_eq!(result.structured()["exact_window_effect"]["verified"], true);
+ assert_eq!(
+ frontmost_pid(),
+ target.pid,
+ "independent frontmost-process oracle"
+ );
+ assert_eq!(
+ focused_window_title(target.pid),
+ SECONDARY_TITLE,
+ "independent AX key-window oracle"
+ );
+ assert_eq!(
+ layer_zero_front().id,
+ secondary,
+ "independent CGWindow z-order oracle"
+ );
+
+ assert_ne!(focused_window_title(target.pid), MAIN_TITLE);
+
+ let recovered = driver.call(
+ "bring_to_front",
+ serde_json::json!({"pid": occluder.pid, "window_id": occluder_main}),
+ );
+ assert!(
+ !recovered.is_error(),
+ "prior app recovery failed: {}",
+ recovered.text()
+ );
+ assert_eq!(frontmost_pid(), occluder.pid);
+ assert_eq!(layer_zero_front().id, occluder_main);
+}
+
+#[test]
+#[ignore]
+fn modal_sheet_never_yields_false_parent_success() {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named("macos-bring-to-front-sheet")
+ .expect("start installed macOS daemon proxy");
+ let target = launch(&mut driver, Some("sheet"));
+ let windows = wait_for_window_ids(&target, &["main", "secondary", "sheet"]);
+ let parent = windows["main"];
+ let result = driver.call(
+ "bring_to_front",
+ serde_json::json!({"pid": target.pid, "window_id": parent}),
+ );
+ assert!(
+ result.is_error(),
+ "modal parent falsely reported success: {}",
+ result.raw
+ );
+ assert_eq!(result.structured()["activated"], false);
+ assert_ne!(focused_window_title(target.pid), MAIN_TITLE);
+}
+
+#[test]
+#[ignore]
+fn floating_panel_is_typed_nonordinary_without_focus_theft() {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named("macos-bring-to-front-floating")
+ .expect("start installed macOS daemon proxy");
+ let target = launch(&mut driver, Some("floating"));
+ let windows = wait_for_window_ids(&target, &["main", "secondary", "floating"]);
+ let floating_id = windows["floating"];
+ let floating = cg_windows()
+ .into_iter()
+ .find(|window| window.pid == target.pid && window.id == floating_id)
+ .expect("floating panel in CoreGraphics oracle");
+ assert_ne!(
+ floating.layer, 0,
+ "fixture floating panel unexpectedly ordinary"
+ );
+ let occluder = launch(&mut driver, None);
+ wait_for_window_ids(&occluder, &["main"]);
+ activate_and_raise(occluder.pid, MAIN_TITLE);
+
+ let result = driver.call(
+ "bring_to_front",
+ serde_json::json!({"pid": target.pid, "window_id": floating.id}),
+ );
+ assert!(
+ result.is_error(),
+ "floating panel falsely reported success: {}",
+ result.raw
+ );
+ assert_eq!(
+ result.structured()["code"],
+ "bring_to_front_window_not_ordinary"
+ );
+ assert_eq!(result.structured()["request_accepted"], false);
+ assert_eq!(
+ frontmost_pid(),
+ occluder.pid,
+ "validation stole focus before refusing"
+ );
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
index b8e88edc62..04df21f2ca 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
@@ -423,6 +423,22 @@ pub fn get_process_psn_for_window(window_id: u32, pid: libc::pid_t, out_psn: &mu
false
}
+/// Return whether WindowServer currently considers the exact window's process
+/// frontmost. Unlike `NSWorkspace.frontmostApplication`, this query does not
+/// depend on the caller's AppKit run loop processing an activation update.
+pub fn front_process_matches(target_pid: libc::pid_t, target_wid: u32) -> Option<bool> {
+ let get_front = get_front_process_fn()?;
+ let mut front_psn = [0u8; 8];
+ if unsafe { get_front(front_psn.as_mut_ptr() as *mut c_void) } != 0 {
+ return None;
+ }
+ let mut target_psn = [0u8; 8];
+ if !get_process_psn_for_window(target_wid, target_pid, &mut target_psn) {
+ return None;
+ }
+ Some(front_psn == target_psn)
+}
+
/// Make `target_pid` and `target_wid` WindowServer-frontmost and leave them
/// there. Unlike [`with_foreground_assist`], this deliberately does not save or
/// restore the previous process. It is the persistent counterpart required by
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs
index d2aaa26eee..3c5b4167ca 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs
@@ -1,51 +1,46 @@
//! macOS `bring_to_front`.
//!
-//! `bring_to_front` exists to let an agent pay a one-shot, persistent foreground
-//! swap before driving a focus-proxy target — a window that only accepts input
-//! while its host app genuinely holds activation. The macOS input rungs never
-//! need this internally: every `CGEvent.postToPid` dispatch reaches a
-//! backgrounded window, and the `delivery_mode:"foreground"` rung does its own
-//! sub-millisecond front→act→restore flash. The one surface that flash can't
-//! satisfy is a remote-desktop client (e.g. Microsoft's Windows App / RDP),
-//! which re-establishes its keyboard channel with the remote host *on
-//! activation* and needs the app to stay frontmost across the whole interaction
-//! — not flashed and restored. `bring_to_front` is that explicit, persistent
-//! activation.
-//!
-//! It fronts the exact window through WindowServer's
-//! `SLPSSetFrontProcessWithOptions` and falls back to
-//! `-[NSRunningApplication activateWithOptions:]` when that SPI is unavailable.
-//! Unlike the rest of the macOS driver this DOES steal foreground — it is an
-//! explicit opt-in, never called by the input ladder.
+//! This is the explicit persistent-foreground escape hatch for focus-proxy
+//! applications. A successful native request is only a request receipt: when
+//! an exact `window_id` is supplied, the tool independently verifies the
+//! process, semantic key window, and WindowServer layer-0 order before it says
+//! `activated: true`.
+
+use std::time::{Duration, Instant};
use async_trait::async_trait;
+use core_foundation::base::{CFRelease, CFTypeRef};
use cua_driver_core::{
protocol::ToolResult,
tool::{Tool, ToolDef},
};
use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};
-use serde_json::Value;
+use serde_json::{json, Value};
+
+use crate::ax::bindings::{
+ ax_get_window_id, copy_ax_windows, perform_action, set_bool_attr_true,
+ AXUIElementCreateApplication,
+};
pub struct BringToFrontTool;
static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
+const VERIFY_TIMEOUT: Duration = Duration::from_millis(900);
+const VERIFY_POLL: Duration = Duration::from_millis(20);
+const VERIFY_STABLE: Duration = Duration::from_millis(100);
+
fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
name: "bring_to_front".into(),
- description: "Persistently activate an app so it genuinely holds macOS foreground, \
- then leave it there. Most input does NOT need this — every macOS \
- dispatch reaches backgrounded windows, and `delivery_mode:\"foreground\"` \
- does its own brief front→act→restore. Reach for `bring_to_front` only \
- for a focus-proxy surface that re-arms its own input channel on \
- activation and must stay frontmost across the interaction — chiefly a \
- remote-desktop client (Microsoft Windows App / RDP), where the brief \
- flash drops keystrokes. Fronts the exact macOS window through \
- WindowServer when `window_id` is present, with app-level Cocoa \
- activation as a fallback. This DOES steal foreground — explicit \
- opt-in, never used by the input ladder."
+ description: "Persistently activate an app and leave it in the foreground. Most input \
+ does not need this; use it only for a focus-proxy surface that must remain \
+ foreground across interactions. With window_id, success means the exact ordinary \
+ macOS window was independently verified as the focused window and first in \
+ WindowServer layer-0 order. Request acceptance alone is reported as a partial \
+ result, never as activation. This DOES steal foreground."
.into(),
- input_schema: serde_json::json!({
+ input_schema: json!({
"type": "object",
"required": ["pid"],
"properties": {
@@ -61,6 +56,194 @@ fn def() -> &'static ToolDef {
})
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct ExactWindowObservation {
+ workspace_frontmost_pid: Option<i32>,
+ front_process_matches_target: Option<bool>,
+ focused_window_id: Option<u32>,
+ frontmost_ordinary_window_id: Option<u32>,
+ target_visible_ordinary: bool,
+}
+
+impl ExactWindowObservation {
+ fn frontmost_pid(self, pid: i32) -> Option<i32> {
+ match self.front_process_matches_target {
+ Some(true) => Some(pid),
+ Some(false) => None,
+ None => self.workspace_frontmost_pid,
+ }
+ }
+
+ fn process_activated(self, pid: i32) -> bool {
+ self.frontmost_pid(pid) == Some(pid)
+ }
+
+ fn exact_window_focused(self, window_id: u32) -> bool {
+ self.focused_window_id == Some(window_id)
+ }
+
+ fn exact_window_frontmost_ordinary(self, window_id: u32) -> bool {
+ self.target_visible_ordinary && self.frontmost_ordinary_window_id == Some(window_id)
+ }
+
+ fn exact_postcondition(self, pid: i32, window_id: u32) -> bool {
+ self.process_activated(pid)
+ && self.exact_window_focused(window_id)
+ && self.exact_window_frontmost_ordinary(window_id)
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ExactOutcome {
+ Activated,
+ Partial,
+ Failed,
+}
+
+fn classify_exact_outcome(
+ request_accepted: bool,
+ pid: i32,
+ window_id: u32,
+ observation: ExactWindowObservation,
+) -> ExactOutcome {
+ if observation.exact_postcondition(pid, window_id) {
+ ExactOutcome::Activated
+ } else if request_accepted
+ || observation.process_activated(pid)
+ || observation.exact_window_focused(window_id)
+ || observation.exact_window_frontmost_ordinary(window_id)
+ {
+ ExactOutcome::Partial
+ } else {
+ ExactOutcome::Failed
+ }
+}
+
+fn observe_exact_window(pid: i32, window_id: u32) -> ExactWindowObservation {
+ let windows = crate::windows::visible_windows();
+ let target_visible_ordinary = windows
+ .iter()
+ .any(|window| window.pid == pid && window.window_id == window_id && window.layer == 0);
+ let frontmost_ordinary_window_id = windows
+ .iter()
+ .max_by_key(|window| window.z_index)
+ .map(|window| window.window_id);
+ ExactWindowObservation {
+ workspace_frontmost_pid: crate::apps::frontmost_pid(),
+ front_process_matches_target: crate::input::skylight::front_process_matches(pid, window_id),
+ focused_window_id: crate::ax::bindings::focused_window_id_of_pid(pid),
+ frontmost_ordinary_window_id,
+ target_visible_ordinary,
+ }
+}
+
+fn wait_for_exact_window(pid: i32, window_id: u32) -> ExactWindowObservation {
+ let deadline = Instant::now() + VERIFY_TIMEOUT;
+ let mut stable_since = None;
+ let mut observation;
+ loop {
+ let now = Instant::now();
+ observation = observe_exact_window(pid, window_id);
+ if observation.exact_postcondition(pid, window_id) {
+ let since = stable_since.get_or_insert(now);
+ if now.duration_since(*since) >= VERIFY_STABLE {
+ return observation;
+ }
+ } else {
+ stable_since = None;
+ }
+ if now >= deadline {
+ return observation;
+ }
+ std::thread::sleep(VERIFY_POLL);
+ }
+}
+
+/// Best-effort completion of the one exact-window request. This addresses only
+/// the requested AX window; it never orders every window owned by the process.
+fn raise_exact_ax_window(pid: i32, window_id: u32) -> bool {
+ unsafe {
+ let app = AXUIElementCreateApplication(pid);
+ if app.is_null() {
+ return false;
+ }
+ let mut target = None;
+ for window in copy_ax_windows(app) {
+ if target.is_none() && ax_get_window_id(window) == Some(window_id) {
+ target = Some(window);
+ } else {
+ CFRelease(window as CFTypeRef);
+ }
+ }
+ CFRelease(app as CFTypeRef);
+ let Some(target) = target else {
+ return false;
+ };
+ let raised = perform_action(target, "AXRaise") == 0;
+ let main = set_bool_attr_true(target, "AXMain") == 0;
+ let focused = set_bool_attr_true(target, "AXFocused") == 0;
+ CFRelease(target as CFTypeRef);
+ raised || main || focused
+ }
+}
+
+fn exact_result(
+ pid: i32,
+ window_id: u32,
+ path: &'static str,
+ request_accepted: bool,
+ observation: ExactWindowObservation,
+) -> ToolResult {
+ let outcome = classify_exact_outcome(request_accepted, pid, window_id, observation);
+ let activated = outcome == ExactOutcome::Activated;
+ let process_activated = observation.process_activated(pid);
+ let frontmost_pid = observation.frontmost_pid(pid);
+ let exact_window_focused = observation.exact_window_focused(window_id);
+ let exact_window_frontmost_ordinary = observation.exact_window_frontmost_ordinary(window_id);
+ let status = match outcome {
+ ExactOutcome::Activated => "activated",
+ ExactOutcome::Partial => "partial",
+ ExactOutcome::Failed => "failed",
+ };
+ let structured = json!({
+ "status": status,
+ "code": if activated { "bring_to_front_exact_window_verified" } else { "bring_to_front_exact_window_unverified" },
+ "pid": pid,
+ "window_id": window_id,
+ "activated": activated,
+ "path": path,
+ "request_accepted": request_accepted,
+ "process_activated": process_activated,
+ "exact_window_effect": {
+ "verified": activated,
+ "focused": exact_window_focused,
+ "frontmost_ordinary": exact_window_frontmost_ordinary,
+ "target_visible_ordinary": observation.target_visible_ordinary,
+ },
+ "observed": {
+ "frontmost_pid": frontmost_pid,
+ "workspace_frontmost_pid": observation.workspace_frontmost_pid,
+ "front_process_matches_target": observation.front_process_matches_target,
+ "focused_window_id": observation.focused_window_id,
+ "frontmost_ordinary_window_id": observation.frontmost_ordinary_window_id,
+ }
+ });
+ if activated {
+ ToolResult::text(format!(
+ "Brought exact window {window_id} for pid {pid} to the foreground."
+ ))
+ .with_structured(structured)
+ } else {
+ ToolResult::error(format!(
+ "bring_to_front: exact window {window_id} for pid {pid} was not verified \
+ as frontmost and focused (request_accepted={request_accepted}, \
+ process_activated={process_activated}, focused={exact_window_focused}, \
+ frontmost_ordinary={exact_window_frontmost_ordinary})."
+ ))
+ .with_structured(structured)
+ }
+}
+
#[async_trait]
impl Tool for BringToFrontTool {
fn def(&self) -> &ToolDef {
@@ -75,68 +258,337 @@ impl Tool for BringToFrontTool {
return ToolResult::error(format!(
"bring_to_front: `pid` {p} is out of range for a process identifier."
))
- .with_structured(serde_json::json!({
+ .with_structured(json!({
"code": "bring_to_front_pid_out_of_range",
"pid": p,
}));
}
},
- None => return ToolResult::error("Missing required integer field: pid".to_string()),
+ None => return ToolResult::error("Missing required integer field: pid"),
+ };
+ let window_id = match args.get("window_id") {
+ Some(value) => match value.as_i64().and_then(|value| u32::try_from(value).ok()) {
+ Some(window_id) if window_id != 0 => Some(window_id),
+ _ => {
+ return ToolResult::error("bring_to_front: window_id is out of range")
+ .with_structured(json!({
+ "code": "bring_to_front_window_id_out_of_range",
+ "pid": pid,
+ "window_id": value,
+ "activated": false,
+ "request_accepted": false,
+ }));
+ }
+ },
+ None => None,
+ };
+
+ let Some(app) =
+ (unsafe { NSRunningApplication::runningApplicationWithProcessIdentifier(pid) })
+ else {
+ return ToolResult::error(format!(
+ "bring_to_front: no running application for pid {pid} (process not found or exited)."
+ ))
+ .with_structured(json!({
+ "code": "bring_to_front_pid_not_found",
+ "pid": pid,
+ "activated": false,
+ "request_accepted": false,
+ }));
};
- let window_id = args.get("window_id").and_then(Value::as_i64);
-
- let skylight_activated = window_id
- .and_then(|value| u32::try_from(value).ok())
- .is_some_and(|window_id| {
- crate::input::skylight::set_front_process_persistently(pid, window_id)
- });
-
- // Cocoa activation is the public fallback. On recent macOS it may
- // accept a request from a background process without actually changing
- // WindowServer ownership, which is why the exact-window SkyLight path
- // runs first.
- let activation = skylight_activated.then_some(true).or_else(|| unsafe {
- NSRunningApplication::runningApplicationWithProcessIdentifier(pid).map(|app| {
- app.activateWithOptions(
- NSApplicationActivationOptions::NSApplicationActivateAllWindows,
- )
- })
- });
- let activated = match activation {
- None => {
+ if let Some(window_id) = window_id {
+ let Some(window) = crate::windows::window_info_by_id(window_id) else {
return ToolResult::error(format!(
- "bring_to_front: no running application for pid {pid} \
- (process not found or already exited)."
+ "bring_to_front: window_id {window_id} is stale or unknown."
))
- .with_structured(serde_json::json!({
- "code": "bring_to_front_pid_not_found",
+ .with_structured(json!({
+ "code": "bring_to_front_window_not_found",
"pid": pid,
+ "window_id": window_id,
+ "activated": false,
+ "request_accepted": false,
+ }));
+ };
+ if window.pid != pid {
+ return ToolResult::error(format!(
+ "bring_to_front: window_id {window_id} is owned by pid {}, not pid {pid}.",
+ window.pid
+ ))
+ .with_structured(json!({
+ "code": "bring_to_front_window_pid_mismatch",
+ "pid": pid,
+ "window_id": window_id,
+ "owner_pid": window.pid,
+ "activated": false,
+ "request_accepted": false,
+ }));
+ }
+ if window.layer != 0 {
+ return ToolResult::error(format!(
+ "bring_to_front: window_id {window_id} is layer {}, not an ordinary layer-0 window.",
+ window.layer
+ ))
+ .with_structured(json!({
+ "code": "bring_to_front_window_not_ordinary",
+ "pid": pid,
+ "window_id": window_id,
+ "layer": window.layer,
+ "activated": false,
+ "request_accepted": false,
}));
}
- Some(activated) => activated,
- };
- if !activated {
- return ToolResult::error(format!(
- "bring_to_front: macOS rejected activation for pid {pid} \
- (activateWithOptions returned NO — the app may be hidden or \
- terminating, or the system denied the foreground swap)."
+ // The persistent kCPSNoWindows request owns process activation
+ // without broadly ordering every application window. The separate
+ // kCPSUserGenerated sequence then makes only the requested window
+ // native-key. Pair both with public Cocoa activation and re-assert
+ // only the exact AX window below; the three independent
+ // postconditions remain authoritative over every request receipt.
+ let skylight_process_accepted =
+ crate::input::skylight::set_front_process_persistently(pid, window_id);
+ let skylight_exact_accepted =
+ crate::input::skylight::make_exact_window_key(pid, window_id);
+ let cocoa_accepted = unsafe {
+ app.activateWithOptions(
+ NSApplicationActivationOptions::NSApplicationActivateAllWindows,
+ )
+ };
+ let ax_window_requested = raise_exact_ax_window(pid, window_id);
+ let path = match (
+ skylight_process_accepted,
+ skylight_exact_accepted,
+ cocoa_accepted,
+ ax_window_requested,
+ ) {
+ (true, true, true, true) => "skylight_process_exact_cocoa_ax",
+ (true, true, _, _) => "skylight_process_exact",
+ (true, false, _, true) => "skylight_process_ax",
+ (true, false, _, false) => "skylight_process",
+ (false, true, true, true) => "skylight_exact_cocoa_ax",
+ (false, true, _, _) => "skylight_exact",
+ (false, false, true, true) => "cocoa_ax",
+ (false, false, true, false) => "cocoa",
+ (false, false, false, true) => "ax",
+ (false, false, false, false) => "none",
+ };
+ let request_accepted = skylight_process_accepted
+ || skylight_exact_accepted
+ || cocoa_accepted
+ || ax_window_requested;
+ return exact_result(
+ pid,
+ window_id,
+ path,
+ request_accepted,
+ wait_for_exact_window(pid, window_id),
+ );
+ }
+
+ let request_accepted = unsafe {
+ app.activateWithOptions(NSApplicationActivationOptions::NSApplicationActivateAllWindows)
+ };
+ let deadline = Instant::now() + VERIFY_TIMEOUT;
+ let activated = loop {
+ if crate::apps::frontmost_pid() == Some(pid) {
+ break true;
+ }
+ if Instant::now() >= deadline {
+ break false;
+ }
+ std::thread::sleep(VERIFY_POLL);
+ };
+ let structured = json!({
+ "status": if activated { "activated" } else if request_accepted { "partial" } else { "failed" },
+ "code": if activated { "bring_to_front_process_verified" } else { "bring_to_front_process_unverified" },
+ "pid": pid,
+ "window_id": Value::Null,
+ "activated": activated,
+ "path": "cocoa",
+ "request_accepted": request_accepted,
+ "process_activated": activated,
+ });
+ if activated {
+ ToolResult::text(format!("Brought pid {pid} to the foreground."))
+ .with_structured(structured)
+ } else {
+ ToolResult::error(format!(
+ "bring_to_front: pid {pid} did not become frontmost (request_accepted={request_accepted})."
))
- .with_structured(serde_json::json!({
- "code": "bring_to_front_activation_rejected",
- "pid": pid,
- "activated": false,
- }));
+ .with_structured(structured)
}
+ }
+}
- ToolResult::text(format!("Brought pid {pid} to the foreground.")).with_structured(
- serde_json::json!({
- "pid": pid,
- "window_id": window_id,
- "activated": true,
- "path": if skylight_activated { "skylight" } else { "cocoa" },
- }),
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn observation(
+ workspace_frontmost_pid: Option<i32>,
+ front_process_matches_target: Option<bool>,
+ focused_window_id: Option<u32>,
+ frontmost_ordinary_window_id: Option<u32>,
+ target_visible_ordinary: bool,
+ ) -> ExactWindowObservation {
+ ExactWindowObservation {
+ workspace_frontmost_pid,
+ front_process_matches_target,
+ focused_window_id,
+ frontmost_ordinary_window_id,
+ target_visible_ordinary,
+ }
+ }
+
+ #[test]
+ fn exact_success_requires_process_focus_and_layer_zero_order() {
+ assert_eq!(
+ classify_exact_outcome(
+ true,
+ 42,
+ 7,
+ observation(Some(9), Some(true), Some(7), Some(7), true),
+ ),
+ ExactOutcome::Activated
+ );
+ for incomplete in [
+ observation(Some(42), Some(false), Some(7), Some(7), true),
+ observation(Some(42), Some(true), Some(8), Some(7), true),
+ observation(Some(42), Some(true), Some(7), Some(8), true),
+ observation(Some(42), Some(true), Some(7), Some(7), false),
+ ] {
+ assert_eq!(
+ classify_exact_outcome(true, 42, 7, incomplete),
+ ExactOutcome::Partial
+ );
+ }
+ }
+
+ #[test]
+ fn process_oracle_falls_back_to_workspace_only_when_skylight_is_unavailable() {
+ assert_eq!(
+ classify_exact_outcome(
+ true,
+ 42,
+ 7,
+ observation(Some(42), None, Some(7), Some(7), true),
+ ),
+ ExactOutcome::Activated
+ );
+ assert_eq!(
+ classify_exact_outcome(
+ true,
+ 42,
+ 7,
+ observation(Some(9), None, Some(7), Some(7), true),
+ ),
+ ExactOutcome::Partial
+ );
+ }
+
+ #[test]
+ fn request_acceptance_or_z_order_alone_is_only_partial() {
+ assert_eq!(
+ classify_exact_outcome(
+ true,
+ 42,
+ 7,
+ observation(Some(42), Some(true), Some(8), Some(7), true),
+ ),
+ ExactOutcome::Partial
+ );
+ assert_eq!(
+ classify_exact_outcome(
+ false,
+ 42,
+ 7,
+ observation(None, Some(false), None, Some(7), true),
+ ),
+ ExactOutcome::Partial
+ );
+ }
+
+ #[test]
+ fn no_request_and_no_observed_effect_is_failed() {
+ assert_eq!(
+ classify_exact_outcome(
+ false,
+ 42,
+ 7,
+ observation(Some(9), Some(false), Some(8), Some(8), true),
+ ),
+ ExactOutcome::Failed
+ );
+ }
+
+ #[test]
+ fn partial_result_keeps_request_process_and_exact_effect_distinct() {
+ let result = exact_result(
+ 42,
+ 7,
+ "skylight_ax",
+ true,
+ observation(Some(9), Some(true), Some(8), Some(7), true),
+ );
+ assert_eq!(result.is_error, Some(true));
+ let structured = result.structured_content.expect("structured result");
+ assert_eq!(structured["status"], "partial");
+ assert_eq!(structured["activated"], false);
+ assert_eq!(structured["request_accepted"], true);
+ assert_eq!(structured["process_activated"], true);
+ assert_eq!(structured["observed"]["frontmost_pid"], 42);
+ assert_eq!(structured["observed"]["workspace_frontmost_pid"], 9);
+ assert_eq!(structured["observed"]["front_process_matches_target"], true);
+ assert_eq!(structured["exact_window_effect"]["focused"], false);
+ assert_eq!(
+ structured["exact_window_effect"]["frontmost_ordinary"],
+ true
+ );
+ assert_eq!(structured["exact_window_effect"]["verified"], false);
+ }
+
+ #[test]
+ fn structured_frontmost_pid_never_echoes_stale_workspace_state_as_authoritative() {
+ let definite_negative = exact_result(
+ 42,
+ 7,
+ "skylight_ax",
+ true,
+ observation(Some(42), Some(false), Some(7), Some(7), true),
+ )
+ .structured_content
+ .expect("structured negative result");
+ assert_eq!(definite_negative["process_activated"], false);
+ assert_eq!(definite_negative["observed"]["frontmost_pid"], Value::Null);
+ assert_eq!(definite_negative["observed"]["workspace_frontmost_pid"], 42);
+
+ let fallback = exact_result(
+ 42,
+ 7,
+ "cocoa_ax",
+ true,
+ observation(Some(42), None, Some(7), Some(7), true),
)
+ .structured_content
+ .expect("structured fallback result");
+ assert_eq!(fallback["process_activated"], true);
+ assert_eq!(fallback["observed"]["frontmost_pid"], 42);
+ assert_eq!(fallback["observed"]["workspace_frontmost_pid"], 42);
+ }
+
+ #[test]
+ fn verified_result_is_the_only_non_error_exact_mapping() {
+ let result = exact_result(
+ 42,
+ 7,
+ "skylight_ax",
+ true,
+ observation(Some(9), Some(true), Some(7), Some(7), true),
+ );
+ assert_eq!(result.is_error, None);
+ let structured = result.structured_content.expect("structured result");
+ assert_eq!(structured["status"], "activated");
+ assert_eq!(structured["activated"], true);
+ assert_eq!(structured["exact_window_effect"]["verified"], true);
}
}
diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
index 7251cd8f49..34712a9569 100644
--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
@@ -51,6 +51,9 @@ let kScrollTopMarker = "SCROLL_TOP_MARKER_v1"
let kScrollBottomMarker = "SCROLL_BOTTOM_MARKER_v1"
let kExitButtonAID = "btn-exit"
let kMenuItemTitle = "Harness Test Item"
+let kSecondaryWindowTitle = "CuaTestHarness AppKit Secondary"
+let kSheetWindowTitle = "CuaTestHarness AppKit Sheet"
+let kFloatingWindowTitle = "CuaTestHarness AppKit Floating"
// MARK: - Controller
@@ -499,6 +502,70 @@ final class ClickTargetButton: NSButton {
override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
}
+// Opt-in windows for the persistent exact-window activation certification.
+// Ordinary harness launches remain byte-for-byte and behaviorally unchanged.
+final class BringToFrontMatrixWindows {
+ let secondary: NSWindow
+ var sheet: NSWindow?
+ var floating: NSPanel?
+
+ init(parent: NSWindow, mode: String) {
+ secondary = NSWindow(
+ contentRect: NSRect(x: 40, y: 40, width: 420, height: 240),
+ styleMask: [.titled, .closable], backing: .buffered, defer: false)
+ secondary.title = kSecondaryWindowTitle
+ secondary.isReleasedWhenClosed = false
+ secondary.isRestorable = false
+ secondary.contentView = NSTextField(labelWithString: "bring_to_front secondary ordinary window")
+ secondary.orderFront(nil)
+
+ if mode == "sheet" {
+ let candidate = NSWindow(
+ contentRect: NSRect(x: 0, y: 0, width: 360, height: 160),
+ styleMask: [.titled], backing: .buffered, defer: false)
+ candidate.title = kSheetWindowTitle
+ candidate.contentView = NSTextField(labelWithString: "modal sheet blocks parent key status")
+ sheet = candidate
+ parent.beginSheet(candidate)
+ } else if mode == "floating" {
+ let candidate = NSPanel(
+ contentRect: NSRect(x: 180, y: 180, width: 320, height: 140),
+ styleMask: [.titled, .utilityWindow], backing: .buffered, defer: false)
+ candidate.title = kFloatingWindowTitle
+ candidate.level = .floating
+ candidate.isFloatingPanel = true
+ candidate.contentView = NSTextField(labelWithString: "floating accessory panel")
+ candidate.orderFront(nil)
+ floating = candidate
+ }
+ }
+}
+
+func writeBringToFrontWindowReport(
+ main: NSWindow,
+ matrix: BringToFrontMatrixWindows?
+) {
+ guard let path = ProcessInfo.processInfo.environment["CUA_HARNESS_WINDOW_REPORT"] else {
+ return
+ }
+ var lines = ["main=\(main.windowNumber)"]
+ if let matrix {
+ lines.append("secondary=\(matrix.secondary.windowNumber)")
+ if let sheet = matrix.sheet {
+ lines.append("sheet=\(sheet.windowNumber)")
+ }
+ if let floating = matrix.floating {
+ lines.append("floating=\(floating.windowNumber)")
+ }
+ }
+ do {
+ try (lines.joined(separator: "\n") + "\n").write(
+ toFile: path, atomically: true, encoding: .utf8)
+ } catch {
+ fputs("failed to write window report: \(error)\n", stderr)
+ }
+}
+
// MARK: - Menu bar (Mac-specific scenario: ns_menubar)
func installMenuBar(target: HarnessWindowController) {
@@ -545,7 +612,13 @@ struct CuaAppKitHarness {
let controller = HarnessWindowController()
installMenuBar(target: controller)
controller.show()
+ var matrixWindows: BringToFrontMatrixWindows?
+ if let mode = ProcessInfo.processInfo.environment["CUA_HARNESS_BRING_TO_FRONT_MODE"] {
+ matrixWindows = BringToFrontMatrixWindows(parent: controller.window, mode: mode)
+ }
app.activate(ignoringOtherApps: true)
+ writeBringToFrontWindowReport(main: controller.window, matrix: matrixWindows)
app.run()
+ _ = matrixWindows
}
}