Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): show active-tab browser cursors on macOS (#2493)

Francesco Bonaccic7675a04d5d3
diff --git a/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md b/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md
index 78f24d82b4..16b267f61a 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md
@@ -31,6 +31,27 @@ Use one explicit `session` value throughout. Never substitute a raw CDP
target id, tab ordinal, URL match, or remembered ref for a capability returned
by `get_browser_state`.
+### macOS recording feedback
+
+On macOS, ref- and coordinate-targeted browser mutations now drive the same
+session-scoped agent cursor overlay as native window actions. `browser_click`
+and click-like pointer actions glide to the live page target and pulse;
+`browser_type` glides to and pulses the editable target; hover and scroll glide
+without a click pulse. This feedback is visual-only: it never moves the user's
+physical pointer, changes focus or z-order, or substitutes for CDP delivery.
+
+The driver rechecks the live page visibility over CDP before every visual
+action. An unselected tab remains fully addressable, but its session cursor is
+hidden. When the selected tab acts, its cursor becomes the only browser-session
+cursor shown for that native window. Use one declared session per tab when a
+recording should give tabs stable, distinct cursor colors.
+
+The overlay is emitted only when the page point can be mapped safely into the
+exact bound native window. In particular, unprovable child-frame coordinates
+are skipped rather than drawn in the wrong place. `browser_navigate` has no
+page target, so it intentionally does not invent cursor motion; use a textual
+recording overlay to explain navigation in a public demo.
+
## 1. Select an exact native window
Start or discover the app with the native tools and select one returned
diff --git a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
index 0806245d51..7eb092be0e 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
@@ -514,6 +514,17 @@ background delivery. Use `input_route:"dom_event"` only when synthetic click
semantics are acceptable. Embedded Electron has a separately bounded route;
do not infer that route for arbitrary WKWebView or Tauri hosts.
+For a declared session, typed browser click, type, and pointer mutations animate
+the macOS agent-cursor overlay at the live main-page target. This is a
+recording/observation aid only: the physical pointer stays put and the overlay
+does not deliver input or foreground Chrome. Navigation itself has no pointer
+target and therefore does not synthesize cursor motion.
+
+Before drawing, the browser route checks the target page's live visibility
+without selecting it. An inactive tab's cursor stays hidden. For concurrent
+recordings, assign one session to each tab; the active tab's session color is
+the only browser cursor displayed in that Chrome window.
+
Browser chrome, permission prompts, downloads, file pickers, Safari, Firefox,
and unbound embedded webviews remain native surfaces. Inspect them with
`get_window_state` and use the AX/PX ladder in this file. The legacy `page`
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
index bdbdd3f167..1f26be508f 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
@@ -39,7 +39,10 @@ use super::binding::{
use super::cdp_ws::{CdpConnection, CdpPool};
use super::grant::{ExistingProfileGrant, ExistingProfileGrants, GrantLookup};
use super::mutation::{MutationGates, MutationKey};
-use super::platform::{BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform};
+use super::platform::{
+ BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform, BrowserVisualAction,
+ BrowserVisualActionKind,
+};
use super::prepare::ManagedBrowsers;
use super::reconnect::ReconnectGates;
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
@@ -148,10 +151,58 @@ pub(crate) struct ValidatedTab {
pub conn: Arc<CdpConnection>,
pub record: TargetRecord,
pub tab: TabRecord,
+ /// Live native metadata from this mutation's revalidation, rather than
+ /// the bind-time geometry retained in `record`.
+ pub native: NativeWindowInfo,
/// Flattened CDP session id attached to the tab's target.
pub cdp_session: String,
}
+fn viewport_point_to_screen(
+ native: Rect,
+ metrics: &Value,
+ viewport_x: f64,
+ viewport_y: f64,
+) -> Option<(f64, f64)> {
+ if !viewport_x.is_finite() || !viewport_y.is_finite() {
+ return None;
+ }
+ let viewport = metrics
+ .get("cssVisualViewport")
+ .or_else(|| metrics.get("cssLayoutViewport"))?;
+ let width = viewport.get("clientWidth")?.as_f64()?;
+ let height = viewport.get("clientHeight")?.as_f64()?;
+ if !width.is_finite()
+ || !height.is_finite()
+ || width <= 0.0
+ || height <= 0.0
+ || viewport_x < 0.0
+ || viewport_y < 0.0
+ || viewport_x > width
+ || viewport_y > height
+ {
+ return None;
+ }
+
+ // CDP viewport coordinates and native/CDP window bounds are all DIPs.
+ // Chromium centers the content viewport horizontally and places its
+ // browser chrome above it. A negative inset means the geometry is not
+ // trustworthy enough for visual feedback, so skip rather than mislead.
+ let horizontal_inset = (native.width - width) / 2.0;
+ let top_inset = native.height - height;
+ if !horizontal_inset.is_finite()
+ || !top_inset.is_finite()
+ || horizontal_inset < -1.0
+ || top_inset < -1.0
+ {
+ return None;
+ }
+ Some((
+ native.x + horizontal_inset.max(0.0) + viewport_x,
+ native.y + top_inset.max(0.0) + viewport_y,
+ ))
+}
+
/// Whether a CDP error is Chromium's "method not implemented" shape.
/// Everything else stays a hard failure — a transient error must never
/// be misread as a capability gap.
@@ -1346,10 +1397,80 @@ impl BrowserEngine {
conn,
record,
tab,
+ native,
cdp_session,
})
}
+ /// Animate platform-owned browser feedback without coupling it to input
+ /// delivery. Only main-frame viewport points are mapped: child-frame CDP
+ /// coordinates are not necessarily in the top-level viewport space, and a
+ /// misleading cursor is worse than no cursor.
+ pub(crate) async fn visualize_browser_action(
+ &self,
+ session: &str,
+ validated: &ValidatedTab,
+ cdp_session: &str,
+ viewport_x: f64,
+ viewport_y: f64,
+ kind: BrowserVisualActionKind,
+ ) {
+ // `document.visibilityState` distinguishes the selected tab without
+ // focusing its native window or invoking any CDP activation command.
+ // Treat an unavailable or malformed proof as inactive: omitting
+ // feedback is safer than drawing a cursor over another tab.
+ let tab_is_active = validated
+ .conn
+ .call(
+ Some(&validated.cdp_session),
+ "Runtime.evaluate",
+ json!({
+ "expression": "document.visibilityState === 'visible'",
+ "returnByValue": true,
+ "awaitPromise": false,
+ }),
+ )
+ .await
+ .ok()
+ .and_then(|result| result.pointer("/result/value").and_then(Value::as_bool))
+ .unwrap_or(false);
+
+ let screen_point = if cdp_session == validated.cdp_session {
+ validated
+ .conn
+ .call(Some(cdp_session), "Page.getLayoutMetrics", json!({}))
+ .await
+ .ok()
+ .and_then(|metrics| {
+ viewport_point_to_screen(
+ validated.native.bounds,
+ &metrics,
+ viewport_x,
+ viewport_y,
+ )
+ })
+ } else {
+ // Child-frame coordinates are not necessarily in the top-level
+ // viewport. Still update active-tab gating, but do not guess a
+ // pointer position.
+ None
+ };
+ let (screen_x, screen_y) = screen_point
+ .map(|(x, y)| (Some(x), Some(y)))
+ .unwrap_or((None, None));
+ self.platform
+ .visualize_browser_action(BrowserVisualAction {
+ session: session.to_owned(),
+ window_id: validated.native.window_id,
+ cdp_target_id: validated.tab.cdp_target_id.clone(),
+ tab_is_active,
+ screen_x,
+ screen_y,
+ kind,
+ })
+ .await;
+ }
+
/// Serialize the full revalidate-dispatch-verify interval by the real CDP
/// target rather than by a caller-controlled session id.
pub(crate) async fn lock_mutation(
@@ -2514,6 +2635,45 @@ fn collect_interactive(
mod tests {
use super::*;
+ #[test]
+ fn viewport_point_maps_below_browser_chrome_in_live_native_bounds() {
+ let point = viewport_point_to_screen(
+ Rect::new(100.0, 50.0, 1000.0, 800.0),
+ &json!({
+ "cssVisualViewport": {
+ "clientWidth": 980.0,
+ "clientHeight": 700.0
+ }
+ }),
+ 240.0,
+ 120.0,
+ );
+ assert_eq!(point, Some((350.0, 270.0)));
+ }
+
+ #[test]
+ fn viewport_point_refuses_untrustworthy_or_out_of_view_geometry() {
+ let native = Rect::new(0.0, 0.0, 800.0, 600.0);
+ assert_eq!(
+ viewport_point_to_screen(
+ native,
+ &json!({"cssVisualViewport":{"clientWidth":900.0,"clientHeight":600.0}}),
+ 10.0,
+ 10.0,
+ ),
+ None
+ );
+ assert_eq!(
+ viewport_point_to_screen(
+ native,
+ &json!({"cssVisualViewport":{"clientWidth":800.0,"clientHeight":500.0}}),
+ 801.0,
+ 10.0,
+ ),
+ None
+ );
+ }
+
fn fixture_doc() -> Value {
json!({
"nodeType": 9,
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mod.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mod.rs
index cbd3ccb64d..7bcbe2ef1b 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mod.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mod.rs
@@ -59,10 +59,10 @@ mod v2_tests;
pub use engine::BrowserEngine;
pub use platform::{
- BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform, ExistingProfileSetupOutcome,
- ExistingProfileSetupRequest, PrepareAction, PrepareAttachment, PrepareAttachmentKind,
- PrepareAuthorization, PrepareOutcome, PrepareProfile, PrepareProfileMode, PrepareRequest,
- PrepareSideEffects, PrepareStrategy,
+ BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform, BrowserVisualAction,
+ BrowserVisualActionKind, ExistingProfileSetupOutcome, ExistingProfileSetupRequest,
+ PrepareAction, PrepareAttachment, PrepareAttachmentKind, PrepareAuthorization, PrepareOutcome,
+ PrepareProfile, PrepareProfileMode, PrepareRequest, PrepareSideEffects, PrepareStrategy,
};
pub use refusal::{BrowserRefusal, BrowserRefusalCode};
pub use setup_descriptor::{
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/platform.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/platform.rs
index db4e98215f..fbc48e25cf 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/platform.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/platform.rs
@@ -155,6 +155,42 @@ pub enum BrowserConsentOutcome {
NotPresent,
}
+/// Visual-only feedback for an already-authorized browser mutation.
+///
+/// This is deliberately separate from input delivery: platform adapters may
+/// animate an agent cursor, but must never synthesize input, activate a
+/// browser, or change whether the browser action succeeds.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BrowserVisualActionKind {
+ Click,
+ Type,
+ Hover,
+ RightClick,
+ DoubleClick,
+ Scroll,
+ Drag,
+}
+
+#[derive(Debug, Clone)]
+pub struct BrowserVisualAction {
+ pub session: String,
+ pub window_id: u64,
+ /// Real CDP page target that owns this cursor. Platform adapters may use
+ /// it to keep several session cursors associated with separate tabs.
+ pub cdp_target_id: String,
+ /// Live, non-activating page-visibility proof collected immediately
+ /// before the action. False also covers an unavailable proof so visual
+ /// feedback fails closed instead of appearing over the wrong tab.
+ pub tab_is_active: bool,
+ /// Screen coordinates in the platform-normalized device-independent
+ /// space used by [`NativeWindowInfo::bounds`]. Child-frame or
+ /// untrustworthy geometry intentionally produces no visual point while
+ /// still allowing the platform to update active-tab visibility.
+ pub screen_x: Option<f64>,
+ pub screen_y: Option<f64>,
+ pub kind: BrowserVisualActionKind,
+}
+
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PrepareSideEffects {
pub launched_browser: bool,
@@ -189,6 +225,12 @@ pub trait BrowserPlatform: Send + Sync {
None
}
+ /// Best-effort, visual-only feedback for an authorized browser action.
+ /// The default is a no-op so platforms without an agent-cursor overlay do
+ /// not change behavior. Implementations must not deliver input or alter
+ /// focus/z-order; failures are intentionally not part of browser results.
+ async fn visualize_browser_action(&self, _action: BrowserVisualAction) {}
+
/// Classify `pid`: is it a browser, which engine family, can it do
/// CDP at all. Must not have side effects.
async fn classify_browser(&self, pid: i64) -> Result<BrowserClassification, BrowserRefusal>;
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs
index 06b4feda72..a836d9cc90 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs
@@ -18,6 +18,7 @@ use crate::tool_args::ArgsExt;
use super::cdp_ws::CdpConnection;
use super::engine::{BrowserEngine, ValidatedTab};
+use super::platform::BrowserVisualActionKind;
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
use super::store::{BrowserActionKind, FrameKind, FrameRef};
@@ -55,6 +56,16 @@ impl PointerAction {
}
}
+fn visual_kind(action: PointerAction) -> BrowserVisualActionKind {
+ match action {
+ PointerAction::Hover => BrowserVisualActionKind::Hover,
+ PointerAction::RightClick => BrowserVisualActionKind::RightClick,
+ PointerAction::DoubleClick => BrowserVisualActionKind::DoubleClick,
+ PointerAction::Scroll => BrowserVisualActionKind::Scroll,
+ PointerAction::Drag => BrowserVisualActionKind::Drag,
+ }
+}
+
fn ref_declares_pointer_action(actions: &[BrowserActionKind], action: PointerAction) -> bool {
match action {
PointerAction::Scroll => actions
@@ -381,6 +392,7 @@ impl BrowserPointerTool {
async fn dom_event(
&self,
+ session: &str,
request: &PointerRequest,
validated: &ValidatedTab,
origin: &ResolvedRef,
@@ -393,6 +405,19 @@ impl BrowserPointerTool {
Err(result) => return result,
};
+ if let Ok((x, y)) = point_for_ref(conn, &origin.cdp_session, origin.backend_node_id).await {
+ self.engine
+ .visualize_browser_action(
+ session,
+ validated,
+ &origin.cdp_session,
+ x,
+ y,
+ visual_kind(request.action),
+ )
+ .await;
+ }
+
let (function, arguments) = match request.action {
PointerAction::Hover => (
"function() { const o={bubbles:true,composed:true,view:this.ownerDocument.defaultView}; this.dispatchEvent(new PointerEvent('pointerover',o)); this.dispatchEvent(new PointerEvent('pointerenter',{...o,bubbles:false})); this.dispatchEvent(new MouseEvent('mouseover',o)); this.dispatchEvent(new MouseEvent('mouseenter',{...o,bubbles:false})); return true; }",
@@ -530,6 +555,7 @@ impl BrowserPointerTool {
async fn trusted(
&self,
+ session: &str,
request: &PointerRequest,
validated: &ValidatedTab,
origin_ref: Option<&ResolvedRef>,
@@ -561,6 +587,17 @@ impl BrowserPointerTool {
_ => unreachable!("destination resolution matches request"),
};
+ self.engine
+ .visualize_browser_action(
+ session,
+ validated,
+ cdp_session,
+ origin.0,
+ origin.1,
+ visual_kind(request.action),
+ )
+ .await;
+
if let Err(error) = conn
.call(
Some(cdp_session),
@@ -804,6 +841,7 @@ impl Tool for BrowserPointerTool {
match request.route {
InputRoute::DomEvent => {
self.dom_event(
+ &session,
&request,
&validated,
origin_ref.as_ref().expect("dom route requires ref"),
@@ -813,6 +851,7 @@ impl Tool for BrowserPointerTool {
}
InputRoute::Trusted => {
self.trusted(
+ &session,
&request,
&validated,
origin_ref.as_ref(),
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
index d15dc260bb..f4df24a889 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
@@ -17,7 +17,9 @@ use crate::tool_args::ArgsExt;
use super::approval::MCP_HOST_APPROVAL_ARG;
use super::download::BrowserDownloadTool;
use super::engine::{BrowserEngine, BrowserTabScreenshot};
-use super::platform::{PrepareAuthorization, PrepareProfile, PrepareRequest, PrepareStrategy};
+use super::platform::{
+ BrowserVisualActionKind, PrepareAuthorization, PrepareProfile, PrepareRequest, PrepareStrategy,
+};
use super::pointer::BrowserPointerTool;
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
use super::store::BrowserActionKind;
@@ -916,6 +918,36 @@ impl Tool for BrowserClickTool {
.to_tool_result()
}
};
+ // Cursor feedback is best-effort and visual-only. A missing box
+ // must not turn a valid DOM-event action into a refusal.
+ let _ = conn
+ .call(
+ Some(cdp),
+ "DOM.scrollIntoViewIfNeeded",
+ json!({ "backendNodeId": backend }),
+ )
+ .await;
+ if let Ok(box_model) = conn
+ .call(
+ Some(cdp),
+ "DOM.getBoxModel",
+ json!({ "backendNodeId": backend }),
+ )
+ .await
+ {
+ if let Some((x, y)) = quad_center(&box_model) {
+ self.engine
+ .visualize_browser_action(
+ &session,
+ &validated,
+ cdp,
+ x,
+ y,
+ BrowserVisualActionKind::Click,
+ )
+ .await;
+ }
+ }
return match conn
.call(
Some(cdp),
@@ -986,6 +1018,17 @@ impl Tool for BrowserClickTool {
(None, None) => unreachable!("validated above"),
};
+ self.engine
+ .visualize_browser_action(
+ &session,
+ &validated,
+ cdp,
+ x,
+ y,
+ BrowserVisualActionKind::Click,
+ )
+ .await;
+
if let Err(error) = conn
.call(
Some(cdp),
@@ -1287,6 +1330,37 @@ impl Tool for BrowserTypeTool {
.to_tool_result();
}
+ // Give the recording a visual target before text delivery. Keep this
+ // best-effort: editability/input semantics never depend on the overlay.
+ let _ = conn
+ .call(
+ Some(cdp),
+ "DOM.scrollIntoViewIfNeeded",
+ json!({ "backendNodeId": entry.backend_node_id }),
+ )
+ .await;
+ if let Ok(box_model) = conn
+ .call(
+ Some(cdp),
+ "DOM.getBoxModel",
+ json!({ "backendNodeId": entry.backend_node_id }),
+ )
+ .await
+ {
+ if let Some((x, y)) = quad_center(&box_model) {
+ self.engine
+ .visualize_browser_action(
+ &session,
+ &validated,
+ cdp,
+ x,
+ y,
+ BrowserVisualActionKind::Type,
+ )
+ .await;
+ }
+ }
+
let requested_chars = text.chars().count();
let (typed, delivered_chars) = if mode == "insert_text" {
match conn
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
index f58156473b..2015f34fc1 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
@@ -59,6 +59,7 @@ struct FixtureState {
screenshot_data: String,
viewport_css_width: f64,
viewport_css_height: f64,
+ tab_visible: bool,
/// Every incoming CDP call: (sessionId, method, params).
calls: Vec<(Option<String>, String, Value)>,
}
@@ -83,6 +84,7 @@ impl Default for FixtureState {
screenshot_data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZJrAAAAAASUVORK5CYII=".into(),
viewport_css_width: 800.0,
viewport_css_height: 600.0,
+ tab_visible: true,
calls: Vec::new(),
}
}
@@ -531,6 +533,16 @@ fn fixture_handler(state: SharedState) -> MockHandler {
"clientHeight": st.viewport_css_height
}
})),
+ "Runtime.evaluate"
+ if call.params["expression"] == "document.visibilityState === 'visible'" =>
+ {
+ MockReply::ok(json!({
+ "result": {
+ "type": "boolean",
+ "value": st.tab_visible
+ }
+ }))
+ }
"Page.captureScreenshot" if is_tab => {
MockReply::ok(json!({"data": st.screenshot_data.clone()}))
}
@@ -1639,6 +1651,41 @@ async fn navigation_targets_an_inactive_tab_without_activating_it() {
assert!(recorded_calls(&f, "Target.activateTarget").is_empty());
}
+#[tokio::test]
+async fn browser_visual_feedback_probes_live_tab_visibility_without_activating_it() {
+ let f = fixture_with(|state| state.tab_visible = false).await;
+ let (target, tab) = bind(&f).await;
+ let snap = snapshot(&f, &target, &tab).await;
+ let main_ref = ref_of(&snap, "main", "main-btn");
+
+ let result = BrowserClickTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "ref": main_ref,
+ "input_route": "dom_event",
+ "session": SESSION,
+ }))
+ .await;
+
+ assert_eq!(
+ structured(&result)["status"],
+ "ok",
+ "{}",
+ structured(&result)
+ );
+ let visibility = recorded_calls(&f, "Runtime.evaluate");
+ assert_eq!(visibility.len(), 1, "{visibility:?}");
+ assert_eq!(
+ visibility[0].1["expression"],
+ "document.visibilityState === 'visible'"
+ );
+ assert_eq!(visibility[0].1["returnByValue"], true);
+ assert_eq!(visibility[0].1["awaitPromise"], false);
+ assert!(recorded_calls(&f, "Page.bringToFront").is_empty());
+ assert!(recorded_calls(&f, "Target.activateTarget").is_empty());
+}
+
#[tokio::test]
async fn semantic_refs_enforce_declared_action_kinds_before_delivery() {
let f = fixture_with(|st| st.semantic_large_page = true).await;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
index d4e1145c1d..f7ad2366ff 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
@@ -1,15 +1,18 @@
//! macOS identity and endpoint evidence for the first-class browser tools.
+use std::collections::HashMap;
use std::hash::{Hash, Hasher};
use std::path::PathBuf;
use std::process::Stdio;
+use std::sync::{Arc, Mutex};
use std::time::Duration;
use async_trait::async_trait;
use cua_driver_core::browser::existing_profile_setup_descriptor;
use cua_driver_core::browser::platform::{
- BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform, ExistingProfileSetupOutcome,
- ExistingProfileSetupRequest, PrepareAction, PrepareOutcome, PrepareRequest,
+ BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform, BrowserVisualAction,
+ BrowserVisualActionKind, ExistingProfileSetupOutcome, ExistingProfileSetupRequest,
+ PrepareAction, PrepareOutcome, PrepareRequest,
};
use cua_driver_core::browser::refusal::{BrowserRefusal, BrowserRefusalCode};
use cua_driver_core::browser::types::{
@@ -19,8 +22,74 @@ use cua_driver_core::browser::types::{
};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
+#[derive(Clone)]
+pub struct MacOsBrowserPlatform {
+ cursor_registry: Arc<crate::cursor::CursorRegistry>,
+ browser_cursors: Arc<Mutex<BrowserCursorTracker>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct BrowserCursorBinding {
+ window_id: u64,
+ cdp_target_id: String,
+}
+
#[derive(Debug, Default)]
-pub struct MacOsBrowserPlatform;
+struct BrowserCursorTracker {
+ bindings: HashMap<String, BrowserCursorBinding>,
+}
+
+impl BrowserCursorTracker {
+ /// Record the session-to-tab binding and return the exact overlay
+ /// visibility changes needed for this action. An active tab owns the sole
+ /// visible browser cursor in its native window. An inactive tab hides only
+ /// its own cursor and never disturbs the cursor for the selected tab.
+ fn update(
+ &mut self,
+ session: &str,
+ window_id: u64,
+ cdp_target_id: &str,
+ tab_is_active: bool,
+ ) -> Vec<(String, bool)> {
+ self.bindings.insert(
+ session.to_owned(),
+ BrowserCursorBinding {
+ window_id,
+ cdp_target_id: cdp_target_id.to_owned(),
+ },
+ );
+
+ if !tab_is_active {
+ return vec![(session.to_owned(), false)];
+ }
+
+ self.bindings
+ .iter()
+ .filter(|(_, binding)| binding.window_id == window_id)
+ .map(|(key, binding)| {
+ (
+ key.clone(),
+ key == session && binding.cdp_target_id == cdp_target_id,
+ )
+ })
+ .collect()
+ }
+}
+
+impl MacOsBrowserPlatform {
+ pub fn new(cursor_registry: Arc<crate::cursor::CursorRegistry>) -> Self {
+ Self {
+ cursor_registry,
+ browser_cursors: Arc::new(Mutex::new(BrowserCursorTracker::default())),
+ }
+ }
+}
+
+impl Default for MacOsBrowserPlatform {
+ fn default() -> Self {
+ Self::new(Arc::new(crate::cursor::CursorRegistry::new()))
+ }
+}
fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefusal {
BrowserRefusal::new(code, message)
@@ -330,6 +399,76 @@ impl BrowserPlatform for MacOsBrowserPlatform {
Some("Chromium's trusted CDP Input route activates its standalone browser window on macOS")
}
+ async fn visualize_browser_action(&self, action: BrowserVisualAction) {
+ if action.session.is_empty()
+ || action.cdp_target_id.is_empty()
+ || cua_driver_core::session::is_session_ended(&action.session)
+ {
+ return;
+ }
+
+ let visibility_updates = self.browser_cursors.lock().unwrap().update(
+ &action.session,
+ action.window_id,
+ &action.cdp_target_id,
+ action.tab_is_active,
+ );
+ let cursor_enabled = self
+ .cursor_registry
+ .get_or_create(&action.session)
+ .config
+ .enabled;
+ for (key, visible) in visibility_updates {
+ let enabled = if key == action.session {
+ visible && cursor_enabled
+ } else {
+ visible
+ && self
+ .cursor_registry
+ .get(&key)
+ .is_some_and(|state| state.config.enabled)
+ };
+ crate::cursor::overlay::send_command(
+ key,
+ cursor_overlay::OverlayCommand::SetEnabled(enabled),
+ );
+ }
+ if !action.tab_is_active || !cursor_enabled {
+ return;
+ }
+ let (Some(screen_x), Some(screen_y)) = (action.screen_x, action.screen_y) else {
+ return;
+ };
+ if !screen_x.is_finite() || !screen_y.is_finite() {
+ return;
+ }
+
+ crate::cursor::overlay::send_command(
+ action.session.clone(),
+ cursor_overlay::OverlayCommand::PinAbove(action.window_id),
+ );
+ crate::cursor::overlay::animate_cursor_to(action.session.clone(), screen_x, screen_y).await;
+ self.cursor_registry
+ .update_position(&action.session, screen_x, screen_y);
+
+ if matches!(
+ action.kind,
+ BrowserVisualActionKind::Click
+ | BrowserVisualActionKind::Type
+ | BrowserVisualActionKind::RightClick
+ | BrowserVisualActionKind::DoubleClick
+ | BrowserVisualActionKind::Drag
+ ) {
+ crate::cursor::overlay::send_command(
+ action.session,
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: screen_x,
+ y: screen_y,
+ },
+ );
+ }
+ }
+
async fn classify_browser(&self, pid: i64) -> Result<BrowserClassification, BrowserRefusal> {
let (app, fallback_name, fallback_bundle_id) = tokio::task::spawn_blocking(move || {
let app = crate::apps::list_running_apps()
@@ -798,6 +937,80 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn browser_visual_feedback_updates_the_declared_session_cursor() {
+ let registry = Arc::new(crate::cursor::CursorRegistry::new());
+ let platform = MacOsBrowserPlatform::new(registry.clone());
+ platform
+ .visualize_browser_action(BrowserVisualAction {
+ session: "browser-cursor-test".to_owned(),
+ window_id: 77,
+ cdp_target_id: "tab-A".to_owned(),
+ tab_is_active: true,
+ screen_x: Some(321.0),
+ screen_y: Some(456.0),
+ kind: BrowserVisualActionKind::Click,
+ })
+ .await;
+
+ let state = registry
+ .get("browser-cursor-test")
+ .expect("browser action should materialize its session cursor");
+ let position = state.position.expect("browser cursor position");
+ assert_eq!((position.x, position.y), (321.0, 456.0));
+ }
+
+ #[tokio::test]
+ async fn inactive_tab_feedback_materializes_but_does_not_move_its_cursor() {
+ let registry = Arc::new(crate::cursor::CursorRegistry::new());
+ let platform = MacOsBrowserPlatform::new(registry.clone());
+ platform
+ .visualize_browser_action(BrowserVisualAction {
+ session: "browser-cursor-hidden".to_owned(),
+ window_id: 77,
+ cdp_target_id: "tab-hidden".to_owned(),
+ tab_is_active: false,
+ screen_x: Some(321.0),
+ screen_y: Some(456.0),
+ kind: BrowserVisualActionKind::Click,
+ })
+ .await;
+
+ let state = registry
+ .get("browser-cursor-hidden")
+ .expect("browser action should establish its session-to-tab binding");
+ assert!(
+ state.position.is_none(),
+ "an inactive tab must not animate or move its visible cursor"
+ );
+ }
+
+ #[test]
+ fn browser_cursor_tracker_shows_only_the_active_tabs_session_per_window() {
+ let mut tracker = BrowserCursorTracker::default();
+ assert_eq!(
+ tracker.update("session-red", 77, "tab-A", false),
+ vec![("session-red".to_owned(), false)]
+ );
+
+ let first_active = tracker.update("session-red", 77, "tab-A", true);
+ assert_eq!(first_active, vec![("session-red".to_owned(), true)]);
+
+ let second_active = tracker
+ .update("session-blue", 77, "tab-B", true)
+ .into_iter()
+ .collect::<HashMap<_, _>>();
+ assert_eq!(second_active.get("session-red"), Some(&false));
+ assert_eq!(second_active.get("session-blue"), Some(&true));
+
+ let other_window = tracker.update("session-green", 88, "tab-C", true);
+ assert_eq!(
+ other_window,
+ vec![("session-green".to_owned(), true)],
+ "an active tab in another native window must not hide this window"
+ );
+ }
+
#[test]
fn lsof_parser_accepts_only_loopback_listeners() {
let input = "n127.0.0.1:9222\nn*:9333\nn[::1]:9444\nn0.0.0.0:9555\n";
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
index c75c667a40..2edaea50be 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
@@ -543,6 +543,12 @@ unsafe fn run_appkit(_cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
let _: () = msg_send![win, setBackgroundColor: clear];
let _: () = msg_send![win, setHasShadow: false];
let _: () = msg_send![win, setIgnoresMouseEvents: true];
+ // NSWindowSharingReadOnly = 1. AppKit documents this as the default, but
+ // set it explicitly for the transparent agent overlay so ScreenCaptureKit
+ // includes browser-session cursors in Cua Driver recordings. Tahoe can
+ // otherwise show the overlay live while omitting it from an in-process
+ // display recording.
+ let _: () = msg_send![win, setSharingType: 1u64];
// NSNormalWindowLevel = 0. The overlay lives at the normal window level so
// it appears in CGWindowList layer=0 results (which agents inspect via
// list_windows). Z-ordering above the target is managed dynamically via
@@ -823,6 +829,34 @@ fn dispatch_set_layer_contents(layer_ptr: usize, pixmap: tiny_skia::Pixmap) {
///
/// `NSWindowAbove = 1`; `orderWindow:relativeTo:` accepts any CGWindowID as
/// the `relativeTo` argument — it works cross-application via CGS.
+fn target_is_frontmost_visible_window(
+ target_wid: u64,
+ frontmost_pid: Option<i32>,
+ windows: &[crate::windows::WindowInfo],
+) -> bool {
+ let Some(target) = windows
+ .iter()
+ .find(|window| u64::from(window.window_id) == target_wid)
+ else {
+ return false;
+ };
+ if !target.is_on_screen || target.layer != 0 || frontmost_pid != Some(target.pid) {
+ return false;
+ }
+
+ windows
+ .iter()
+ .filter(|window| {
+ window.is_on_screen
+ && window.layer == 0
+ && window.pid == target.pid
+ && window.bounds.width > 1.0
+ && window.bounds.height > 1.0
+ })
+ .max_by_key(|window| window.z_index)
+ .is_some_and(|window| u64::from(window.window_id) == target_wid)
+}
+
fn dispatch_pin_above(win_ptr: usize, target_wid: u64) {
use std::ffi::c_void;
@@ -837,13 +871,27 @@ fn dispatch_pin_above(win_ptr: usize, target_wid: u64) {
}
unsafe extern "C" fn reorder_cb(ctx: *mut c_void) {
- let (win_ptr, target_wid): (usize, u64) = *Box::from_raw(ctx as *mut (usize, u64));
+ let (win_ptr, target_wid, raise_front): (usize, u64, bool) =
+ *Box::from_raw(ctx as *mut (usize, u64, bool));
let win = win_ptr as *mut objc2::runtime::AnyObject;
// NSWindowAbove = 1; relativeTo: takes NSInteger (i64 on 64-bit)
let _: () = objc2::msg_send![win, orderWindow: 1i64 relativeTo: target_wid as i64];
+
+ // Tahoe can leave a normal-level transparent window behind an
+ // already-frontmost cross-process target even after the relative
+ // ordering request. `orderFrontRegardless` does not activate the
+ // driver app. Use it only when the exact target is already the
+ // frontmost visible normal window, so background browser actions do
+ // not put the overlay above the user's foreground app.
+ if raise_front {
+ let _: () = objc2::msg_send![win, orderFrontRegardless];
+ }
}
- let payload = Box::new((win_ptr, target_wid));
+ let windows = crate::windows::visible_windows();
+ let raise_front =
+ target_is_frontmost_visible_window(target_wid, crate::apps::frontmost_pid(), &windows);
+ let payload = Box::new((win_ptr, target_wid, raise_front));
unsafe {
let main_queue = &raw const _dispatch_main_q as *const c_void;
dispatch_async_f(
@@ -989,6 +1037,26 @@ mod tests {
use super::*;
use std::collections::HashMap;
+ fn window(window_id: u32, pid: i32, z_index: usize) -> crate::windows::WindowInfo {
+ crate::windows::WindowInfo {
+ window_id,
+ pid,
+ app_name: format!("app-{pid}"),
+ title: String::new(),
+ bounds: crate::windows::WindowBounds {
+ x: 0.0,
+ y: 0.0,
+ width: 800.0,
+ height: 600.0,
+ },
+ layer: 0,
+ z_index,
+ is_on_screen: true,
+ on_current_space: None,
+ space_ids: None,
+ }
+ }
+
fn empty_map() -> RenderMap {
let mut cursors = IndexMap::new();
cursors.insert(
@@ -1005,6 +1073,31 @@ mod tests {
}
}
+ #[test]
+ fn frontmost_target_can_raise_overlay_without_covering_another_app() {
+ let target_pid = 100;
+ let mut windows = vec![window(10, target_pid, 20), window(11, 200, 10)];
+ // WindowServer may retain another app's window ahead in its global
+ // list; the active app identity is the authoritative cross-app guard.
+ windows[1].z_index = 30;
+ assert!(target_is_frontmost_visible_window(
+ 10,
+ Some(target_pid),
+ &windows,
+ ));
+
+ // A different foreground app blocks the fallback raise.
+ assert!(!target_is_frontmost_visible_window(10, Some(200), &windows,));
+
+ // So does another visible window belonging to the active target app.
+ windows.push(window(13, target_pid, 40));
+ assert!(!target_is_frontmost_visible_window(
+ 10,
+ Some(target_pid),
+ &windows,
+ ));
+ }
+
fn move_msg(key: &str, x: f64, y: f64) -> OverlayMsg {
OverlayMsg::Cmd(KeyedOverlayCommand {
key: key.to_owned(),
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index 34d315d382..045c7d6517 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -581,7 +581,7 @@ pub fn register_all(registry: &mut ToolRegistry, compat: bool) {
page::MacOsPageBackend::new(state.clone()),
))));
let browser_engine = cua_driver_core::browser::BrowserEngine::new(Arc::new(
- crate::browser::MacOsBrowserPlatform,
+ crate::browser::MacOsBrowserPlatform::new(state.cursor_registry.clone()),
));
cua_driver_core::browser::register_browser_tools(&browser_engine, registry);
// Recording / replay + session-lifecycle tools are platform-independent.