Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): reveal the agent cursor for keyboard-first actions (#4287)
Francesco Bonacci2d0431fb0096
diff --git a/libs/cua-driver/docs/cursor-themes.md b/libs/cua-driver/docs/cursor-themes.mdindex a9b71a83a2..7b395589a7 100644--- a/libs/cua-driver/docs/cursor-themes.md+++ b/libs/cua-driver/docs/cursor-themes.md@@ -74,8 +74,14 @@ cua-driver get_agent_cursor_state '{"session":"demo"}'````get_agent_cursor_state` always includes `position`. It is `null` until the-session cursor first moves, then `{"x": ..., "y": ...}` where the platform-tracks session cursor placement.+session cursor first moves, then `{"x": ..., "y": ...}`: the screen point the+cursor tip was last placed on.++Keyboard actions place a named session's cursor before the input.+`press_key`, `hotkey`, and `type_text` put it on the targeted element, else+at its last position, else at the target window's centre, so a session whose+first action is a keyboard action shows its cursor on that window. Anonymous+calls keep their existing cursor behavior.`set_agent_cursor_motion` changes only movement physics and visibility timing.It does not change artwork. The removed `set_agent_cursor_style` operation anddiff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rsindex 7c4afdc718..00304c02d1 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs@@ -13,12 +13,11 @@ use image::RgbaImage;const CELL_ID: &str = "desktop-agent-cursor-showcase-px";const SESSION: &str = "Cursor showcase";-// MoveTo offsets the artwork centre by a 16-point vector at 45 degrees so the-// cursor tip lands on the requested coordinate. Each axis moves by 16/sqrt(2),-// and the session badge follows that artwork centre.-const CURSOR_ANCHOR_OFFSET_MAGNITUDE: f64 = 16.0;+// MoveTo anchors the artwork `POINTER_ANCHOR_OFFSET` points from the requested+// coordinate at 45 degrees so the cursor tip lands on it. Each axis moves by+// that offset over sqrt(2), and the session badge follows the anchor.const CURSOR_ANCHOR_OFFSET_PER_AXIS: f64 =- CURSOR_ANCHOR_OFFSET_MAGNITUDE * std::f64::consts::FRAC_1_SQRT_2;+ cursor_overlay::POINTER_ANCHOR_OFFSET * std::f64::consts::FRAC_1_SQRT_2;const POINTER_ORACLE_RADIUS: f64 = 24.0;const BADGE_CURSOR_EXCLUSION: f64 = 34.0;@@ -191,6 +190,262 @@ fn semantic_cursor_showcase_records_session_and_action_states() {});}+const KEYBOARD_FIRST_CELL_ID: &str = "window-agent-cursor-keyboard-first-placement";+/// Largest distance, per axis and in screen units, between the reported cursor+/// point and the centre of the window's `list_windows` bounds. Each platform+/// reads the centre from its own native geometry (Windows `GetWindowRect`+/// includes the invisible resize border that the DWM frame bounds leave out),+/// so the two can differ by a few units; a cursor left at the pointer, at a+/// stale session point, or unplaced is far outside it.+const WINDOW_CENTRE_TOLERANCE: f64 = 12.0;++/// A named session whose first action is an untargeted `press_key` shows its+/// agent cursor on the target window before the key is delivered: no element,+/// no pixel target, and no remembered position leave the shared keyboard+/// placement policy with the window centre.+#[test]+#[ignore]+fn keyboard_first_session_places_its_cursor_on_the_target_window() {+ let fixture = native_fixture();+ let case = CaseSpec::delivered(+ KEYBOARD_FIRST_CELL_ID,+ fixture.toolkit,+ fixture.toolkit,+ "press_key_cursor_placement",+ Targeting::NotApplicable,+ Delivery::Foreground,+ Scope::Window,+ platform_route(),+ vec![OracleKind::Protocol],+ );+ execute_case(case, |evidence| {+ let mut driver = spawn_driver_named(KEYBOARD_FIRST_CELL_ID);+ *evidence = recording_evidence(driver.recording_dir());+ let (pid, window_id) = launch_native_fixture(&mut driver, &fixture);++ // A per-run name keeps a long-lived daemon from handing this row a+ // cursor position that an earlier run left behind.+ let session = format!("Keyboard-first cursor {}", std::process::id());+ call_ok(+ &mut driver,+ "start_session",+ serde_json::json!({"session": session}),+ );+ call_ok(+ &mut driver,+ "set_agent_cursor_enabled",+ serde_json::json!({"session": session, "enabled": true}),+ );+ let before = agent_cursor_state(&mut driver, &session);+ assert!(+ before["position"].is_null(),+ "the session cursor must start unplaced so press_key is its first placement: {before}"+ );++ driver.start_behavior_recording();+ call_ok(+ &mut driver,+ "press_key",+ serde_json::json!({+ "session": session,+ "pid": pid,+ "window_id": window_id,+ "key": "escape",+ "delivery_mode": "foreground"+ }),+ );++ let after = agent_cursor_state(&mut driver, &session);+ assert_eq!(+ after["enabled"], true,+ "keyboard-first cursor is hidden: {after}"+ );+ let (Some(x), Some(y)) = (+ after["position"]["x"].as_f64(),+ after["position"]["y"].as_f64(),+ ) else {+ panic!("a keyboard-first press_key left the session cursor unplaced: {after}");+ };+ let (centre_x, centre_y) = window_centre(&mut driver, pid, window_id);+ assert!(+ (x - centre_x).abs() <= WINDOW_CENTRE_TOLERANCE+ && (y - centre_y).abs() <= WINDOW_CENTRE_TOLERANCE,+ "keyboard-first cursor landed at ({x:.1}, {y:.1}), expected the target window \+ centre ({centre_x:.1}, {centre_y:.1}) within {WINDOW_CENTRE_TOLERANCE}: {after}"+ );++ call_ok(+ &mut driver,+ "end_session",+ serde_json::json!({"session": session}),+ );+ Observation::delivered(vec![OracleKind::Protocol], Evidence::default())+ });+}++fn agent_cursor_state(driver: &mut McpDriver, session: &str) -> serde_json::Value {+ let response = driver.call(+ "get_agent_cursor_state",+ serde_json::json!({"session": session}),+ );+ assert!(+ !response.is_error(),+ "get_agent_cursor_state failed: {}",+ response.text()+ );+ response.structured().clone()+}++/// Centre of the target window's `list_windows` bounds.+fn window_centre(driver: &mut McpDriver, pid: u32, window_id: u64) -> (f64, f64) {+ let windows = driver.call("list_windows", serde_json::json!({"pid": pid}));+ let window = windows.structured()["windows"]+ .as_array()+ .and_then(|windows| {+ windows+ .iter()+ .find(|window| window["window_id"].as_u64() == Some(window_id))+ })+ .unwrap_or_else(|| panic!("target window {window_id} is not listed: {}", windows.raw))+ .clone();+ let bounds = &window["bounds"];+ let (Some(x), Some(y), Some(width), Some(height)) = (+ bounds["x"].as_f64(),+ bounds["y"].as_f64(),+ bounds["width"].as_f64(),+ bounds["height"].as_f64(),+ ) else {+ panic!("target window has no bounds: {window}");+ };+ assert!(+ width > 0.0 && height > 0.0,+ "target window is empty: {window}"+ );+ (x + width / 2.0, y + height / 2.0)+}++/// The repo-local native harness the platform's native lane stages.+struct NativeFixture {+ toolkit: &'static str,+ executable: std::path::PathBuf,+ title: &'static str,+}++#[cfg(target_os = "macos")]+fn native_fixture() -> NativeFixture {+ let app = std::env::var("HARNESS_APPKIT_APP")+ .map(std::path::PathBuf::from)+ .ok()+ .filter(|path| path.exists())+ .unwrap_or_else(|| {+ cua_driver_testkit::harness_app("harness-appkit", "CuaTestHarness.AppKit.app")+ });+ NativeFixture {+ toolkit: "appkit",+ executable: app.join("Contents/MacOS/CuaTestHarness.AppKit"),+ title: "CuaTestHarness AppKit",+ }+}++#[cfg(target_os = "windows")]+fn native_fixture() -> NativeFixture {+ NativeFixture {+ toolkit: "wpf",+ executable: cua_driver_testkit::harness_app("harness-wpf", "CuaTestHarness.Wpf.exe"),+ title: "CuaTestHarness WPF",+ }+}++#[cfg(target_os = "linux")]+fn native_fixture() -> NativeFixture {+ NativeFixture {+ toolkit: "gtk3",+ executable: std::env::var("HARNESS_GTK3_EXE")+ .map(std::path::PathBuf::from)+ .ok()+ .filter(|path| path.exists())+ .unwrap_or_else(|| {+ cua_driver_testkit::harness_app("harness-gtk3", "CuaTestHarness.Gtk3")+ }),+ title: "CuaTestHarness GTK3",+ }+}++/// Launch the fixture and return the pid and id of its main window.+fn launch_native_fixture(driver: &mut McpDriver, fixture: &NativeFixture) -> (u32, u64) {+ assert!(+ fixture.executable.exists(),+ "required {} harness is missing at {:?}; run the fixture build",+ fixture.toolkit,+ fixture.executable+ );+ let listed_window_ids = |driver: &mut McpDriver| -> std::collections::HashSet<u64> {+ driver+ .call("list_windows", serde_json::json!({}))+ .structured()["windows"]+ .as_array()+ .into_iter()+ .flatten()+ .filter_map(|window| window["window_id"].as_u64())+ .collect()+ };+ let earlier_windows = listed_window_ids(driver);+ let child = cua_driver_testkit::spawn_in_job(+ std::process::Command::new(&fixture.executable)+ .stdout(std::process::Stdio::null())+ .stderr(std::process::Stdio::null()),+ )+ .unwrap_or_else(|error| panic!("launch {} harness: {error}", fixture.toolkit));+ driver.reaper().push(child);++ // Take the new window with the fixture title, whatever process owns it:+ // a toolkit launcher can hand the window to another process, and an+ // earlier fixture window must not be mistaken for this one.+ let deadline = std::time::Instant::now() + Duration::from_secs(20);+ loop {+ let windows = driver.call("list_windows", serde_json::json!({}));+ let found = windows.structured()["windows"]+ .as_array()+ .into_iter()+ .flatten()+ .find(|window| {+ window["title"]+ .as_str()+ .is_some_and(|title| title.contains(fixture.title))+ && window["window_id"]+ .as_u64()+ .is_some_and(|id| !earlier_windows.contains(&id))+ })+ .and_then(|window| {+ Some((+ u32::try_from(window["pid"].as_u64()?).ok()?,+ window["window_id"].as_u64()?,+ ))+ });+ if let Some((pid, window_id)) =+ found.filter(|(pid, window_id)| *pid != 0 && *window_id != 0)+ {+ driver.reaper().track_pid(pid);+ #[cfg(target_os = "macos")]+ while !cua_driver_testkit::observer::macos::application_presented(pid) {+ assert!(+ std::time::Instant::now() < deadline,+ "AppKit harness pid {pid} did not finish launching"+ );+ settle(50);+ }+ settle(500);+ return (pid, window_id);+ }+ assert!(+ std::time::Instant::now() < deadline,+ "{} harness window never appeared",+ fixture.toolkit+ );+ settle(250);+ }+}+fn assert_cursor_and_badge_pixels_changed(baseline: &RgbaImage,cursor_frame: &RgbaImage,@@ -791,14 +1046,18 @@ mod pixel_oracle_tests {}}-#[cfg(target_os = "macos")]fn spawn_driver() -> McpDriver {- McpDriver::spawn_macos_daemon_proxy_named(CELL_ID).expect("start installed macOS daemon proxy")+ spawn_driver_named(CELL_ID)+}++#[cfg(target_os = "macos")]+fn spawn_driver_named(cell_id: &str) -> McpDriver {+ McpDriver::spawn_macos_daemon_proxy_named(cell_id).expect("start installed macOS daemon proxy")}#[cfg(not(target_os = "macos"))]-fn spawn_driver() -> McpDriver {- McpDriver::spawn_named_with_overlay(CELL_ID)+fn spawn_driver_named(cell_id: &str) -> McpDriver {+ McpDriver::spawn_named_with_overlay(cell_id).expect("start source-built driver with native cursor overlay")}diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rsindex 36f37172d2..ce000fd313 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs@@ -408,6 +408,35 @@ pub fn pointer_for_anchor(x: f64, y: f64, heading: f64) -> (f64, f64) {)}+/// Cursor key for a named session's keyboard and text feedback: the explicit+/// `session` label, else the trusted lifecycle `_session_id`. Anonymous calls+/// return `None` and keep their existing behavior.+pub fn named_session_cursor_key(args: &serde_json::Value) -> Option<String> {+ ["session", "_session_id"].into_iter().find_map(|key| {+ args.get(key)+ .and_then(serde_json::Value::as_str)+ .filter(|session| !session.is_empty())+ .map(str::to_owned)+ })+}++/// Where a keyboard or text action places its session cursor: the explicit+/// element or pixel target, else the cursor's remembered position, else the+/// target window's centre, else the current pointer. Non-finite points are+/// skipped so a bad geometry read cannot poison position reuse.+pub fn keyboard_cursor_target(+ explicit: Option<(f64, f64)>,+ remembered: Option<(f64, f64)>,+ window_center: Option<(f64, f64)>,+ current_pointer: Option<(f64, f64)>,+) -> Option<(f64, f64)> {+ let finite = |point: Option<(f64, f64)>| point.filter(|(x, y)| x.is_finite() && y.is_finite());+ finite(explicit)+ .or_else(|| finite(remembered))+ .or_else(|| finite(window_center))+ .or_else(|| finite(current_pointer))+}+/// Build the shared overlay command for one native pointer position.////// Native drag implementations report the actual event coordinate. Anchoring@@ -470,6 +499,48 @@ mod pointer_tracking_tests {assert!(!sibling.get());}+ #[test]+ fn named_sessions_opt_into_keyboard_cursor_positioning() {+ use serde_json::json;+ assert_eq!(+ named_session_cursor_key(&json!({"session": "editing-run"})).as_deref(),+ Some("editing-run")+ );+ assert_eq!(+ named_session_cursor_key(&json!({"_session_id": "implicit"})).as_deref(),+ Some("implicit")+ );+ assert_eq!(+ named_session_cursor_key(&json!({"cursor_id": "legacy"})),+ None+ );+ assert_eq!(named_session_cursor_key(&json!({"session": ""})), None);+ }++ #[test]+ fn keyboard_cursor_uses_explicit_then_remembered_then_window_then_pointer() {+ let explicit = Some((10.0, 20.0));+ let remembered = Some((30.0, 40.0));+ let window = Some((50.0, 60.0));+ let pointer = Some((70.0, 80.0));+ assert_eq!(+ keyboard_cursor_target(explicit, remembered, window, pointer),+ explicit+ );+ assert_eq!(+ keyboard_cursor_target(None, remembered, window, pointer),+ remembered+ );+ assert_eq!(keyboard_cursor_target(None, None, window, pointer), window);+ assert_eq!(keyboard_cursor_target(None, None, None, pointer), pointer);+ assert_eq!(keyboard_cursor_target(None, None, None, None), None);+ assert_eq!(+ keyboard_cursor_target(Some((f64::NAN, 1.0)), Some((12.0, 34.0)), None, None),+ Some((12.0, 34.0)),+ "invalid coordinates must not poison session position reuse"+ );+ }+#[test]fn session_cleanup_removes_named_cursor_but_preserves_anonymous_default() {let registry = CursorRegistry::new();diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rsindex 264c427615..293491bc0d 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs@@ -596,7 +596,10 @@ impl RenderStateCore {}let idle_hide_ms = self.motion.idle_hide_ms;if idle_hide_ms > 0.0 {- let moving = self.path.is_some() || self.spring.is_some() || self.click_t.is_some();+ let moving = self.path.is_some()+ || self.spring.is_some()+ || self.click_t.is_some()+ || self.visual.resolved_action != CursorAction::Idle;if moving {self.idle_secs = 0.0;self.idle_alpha = 1.0;@@ -768,6 +771,10 @@ impl RenderStateCore {target,} => {self.visual.begin(action, delivery, target);+ // A semantic cue re-reveals an idle-faded cursor that already+ // has a position; `tick_idle` keeps it visible until the cue ends.+ self.idle_secs = 0.0;+ self.idle_alpha = 1.0;self.badge_modifiers = if delivery.is_some() || target.is_some() {Some((delivery, target))} else {@@ -1094,6 +1101,43 @@ mod session_badge_and_action_tests {assert_eq!(core.idle_alpha, 1.0);}+ #[test]+ fn semantic_cue_re_reveals_an_idle_hidden_cursor_until_it_ends() {+ let frame = 1.0 / 60.0;+ let mut core = RenderStateCore::new(CursorConfig::default());+ core.motion.idle_hide_ms = 200.0;+ core.apply_command_base(+ OverlayCommand::ClickPulse { x: 40.0, y: 60.0 },+ false,+ false,+ );+ for _ in 0..120 {+ core.tick_motion(frame);+ }+ assert_eq!(core.idle_alpha, 0.0, "the positioned cursor idles out");++ core.apply_command_base(+ OverlayCommand::BeginAction {+ action: CursorAction::Text,+ delivery: None,+ target: None,+ },+ false,+ false,+ );+ assert_eq!(core.idle_alpha, 1.0, "a keyboard cue re-reveals it");+ for _ in 0..120 {+ core.tick_motion(frame);+ }+ assert_eq!(core.idle_alpha, 1.0, "it stays visible while the cue runs");++ core.apply_command_base(OverlayCommand::EndAction(CursorAction::Text), false, false);+ for _ in 0..120 {+ core.tick_motion(frame);+ }+ assert_eq!(core.idle_alpha, 0.0, "it idles out again after the cue");+ }+#[test]fn session_badge_holds_then_fades_once() {let mut core = RenderStateCore::new(CursorConfig::default());diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex c2017c7afb..5858fb1edb 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -5132,28 +5132,7 @@ fn resolve_cursor_key(args: &Value) -> String {/// `_session_id` is trusted lifecycle state and must behave like a public named/// session for cursor ownership.fn named_session_cursor_key(args: &Value) -> Option<String> {- ["session", "_session_id"].into_iter().find_map(|key| {- args.get(key)- .and_then(Value::as_str)- .filter(|session| !session.is_empty())- .map(str::to_owned)- })-}--fn finite_cursor_point(point: Option<(f64, f64)>) -> Option<(f64, f64)> {- point.filter(|(x, y)| x.is_finite() && y.is_finite())-}--fn choose_keyboard_cursor_target(- explicit: Option<(f64, f64)>,- remembered: Option<(f64, f64)>,- window_center: Option<(f64, f64)>,- current_pointer: Option<(f64, f64)>,-) -> Option<(f64, f64)> {- finite_cursor_point(explicit)- .or_else(|| finite_cursor_point(remembered))- .or_else(|| finite_cursor_point(window_center))- .or_else(|| finite_cursor_point(current_pointer))+ cursor_overlay::named_session_cursor_key(args)}fn mouse_hold_json(cursor_id: &str, hold: Option<&MouseHoldState>) -> Value {@@ -5428,7 +5407,7 @@ async fn position_named_session_keyboard_cursor(let fallback = if named_cursor_id.is_some()&& explicit.is_none()- && finite_cursor_point(remembered).is_none()+ && cursor_overlay::keyboard_cursor_target(None, remembered, None, None).is_none(){tokio::task::spawn_blocking(move || {let center = keyboard_window_center(xid);@@ -5442,7 +5421,7 @@ async fn position_named_session_keyboard_cursor(};let Some((sx, sy)) =- choose_keyboard_cursor_target(explicit, remembered, fallback.0, fallback.1)+ cursor_overlay::keyboard_cursor_target(explicit, remembered, fallback.0, fallback.1)else {return;};@@ -12595,7 +12574,9 @@ impl Tool for SetAgentCursorThemeTool {}}-pub struct GetAgentCursorStateV2Tool;+pub struct GetAgentCursorStateV2Tool {+ state: Arc<ToolState>,+}static CURSOR_STATE_V2_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();@@ -12626,11 +12607,19 @@ impl Tool for GetAgentCursorStateV2Tool {.into_iter().flatten().collect();+ // The last point this session's cursor was placed at, `null` until+ // it first moves (same registry source as macOS).+ let position = self+ .state+ .cursor_registry+ .get(&session)+ .and_then(|cursor| cursor.x.zip(cursor.y))+ .map(|(x, y)| json!({"x": x, "y": y}));ToolResult::text(format!("Agent cursor state for session '{session}'.")).with_structured(json!({"session":session,"enabled":enabled,- "position":null,+ "position":position,"theme":{"id":theme_id,"version":version,@@ -14148,7 +14137,9 @@ pub fn build_registry_with_provider(state: state.clone(),}));r.register(Box::new(SetAgentCursorMotionV2Tool));- r.register(Box::new(GetAgentCursorStateV2Tool));+ r.register(Box::new(GetAgentCursorStateV2Tool {+ state: state.clone(),+ }));r.register(Box::new(SetAgentCursorThemeTool {state: state.clone(),}));diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/session_cursor_target_tests.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/session_cursor_target_tests.rsindex 7860153cc5..d5ee4b44a9 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/session_cursor_target_tests.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/session_cursor_target_tests.rs@@ -1,59 +1,6 @@-use super::{- choose_keyboard_cursor_target, cursor_control_scope, named_session_cursor_key,- reveal_pointer_action_for, CursorControlScope, ToolState,-};+use super::{cursor_control_scope, reveal_pointer_action_for, CursorControlScope, ToolState};use serde_json::json;-#[test]-fn lifecycle_owned_sessions_opt_into_keyboard_cursor_positioning() {- assert_eq!(- named_session_cursor_key(&json!({"session": "editing-run"})).as_deref(),- Some("editing-run")- );- assert_eq!(- named_session_cursor_key(&json!({"cursor_id": "legacy"})),- None- );- assert_eq!(- named_session_cursor_key(&json!({"_session_id": "implicit"})).as_deref(),- Some("implicit")- );- assert_eq!(named_session_cursor_key(&json!({})), None);-}--#[test]-fn keyboard_cursor_uses_explicit_then_remembered_then_safe_seed() {- let explicit = Some((10.0, 20.0));- let remembered = Some((30.0, 40.0));- let window_center = Some((50.0, 60.0));- let current_pointer = Some((70.0, 80.0));-- assert_eq!(- choose_keyboard_cursor_target(explicit, remembered, window_center, current_pointer),- explicit- );- assert_eq!(- choose_keyboard_cursor_target(None, remembered, window_center, current_pointer),- remembered- );- assert_eq!(- choose_keyboard_cursor_target(None, None, window_center, current_pointer),- window_center- );- assert_eq!(- choose_keyboard_cursor_target(None, None, None, current_pointer),- current_pointer- );-}--#[test]-fn invalid_coordinates_do_not_poison_session_position_reuse() {- assert_eq!(- choose_keyboard_cursor_target(Some((f64::NAN, 1.0)), Some((12.0, 34.0)), None, None,),- Some((12.0, 34.0))- );-}-#[test]fn real_pointer_control_requires_explicit_desktop_scope() {assert_eq!(cursor_control_scope(&json!({})), CursorControlScope::Agent);diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rsindex 8023a2055b..09e89171c7 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs@@ -24,6 +24,59 @@ pub(crate) fn resolve_cursor_key(args: &Value) -> String {default_cursor_session()}+/// Place and reveal a named session's cursor before keyboard or text input.+///+/// `explicit` is the targeted element's screen centre when the action has+/// one. Without it the cursor reuses its remembered position, then falls back+/// to the target window's centre, so the first keyboard action of a session+/// is visible. Anonymous calls keep their existing behavior. Geometry and+/// overlay failures are observational and never affect the input result.+pub(crate) async fn position_keyboard_cursor(+ state: &ToolState,+ args: &Value,+ window_id: Option<u32>,+ explicit: Option<(f64, f64)>,+) {+ let Some(cursor_key) = cursor_overlay::named_session_cursor_key(args) else {+ return;+ };+ let remembered = state+ .cursor_registry+ .get(&cursor_key)+ .and_then(|cursor| cursor.position.map(|position| (position.x, position.y)));+ let window_center = match window_id {+ Some(wid)+ if cursor_overlay::keyboard_cursor_target(explicit, remembered, None, None)+ .is_none() =>+ {+ tokio::task::spawn_blocking(move || {+ let bounds = crate::windows::window_bounds_by_id(wid)?;+ (bounds.width > 0.0 && bounds.height > 0.0).then_some((+ bounds.x + bounds.width / 2.0,+ bounds.y + bounds.height / 2.0,+ ))+ })+ .await+ .ok()+ .flatten()+ }+ _ => None,+ };+ let Some((x, y)) =+ cursor_overlay::keyboard_cursor_target(explicit, remembered, window_center, None)+ else {+ return;+ };+ if let Some(wid) = window_id {+ crate::cursor::overlay::send_command(+ cursor_key.clone(),+ cursor_overlay::OverlayCommand::PinAbove(u64::from(wid)),+ );+ }+ crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), x, y).await;+ state.cursor_registry.update_position(&cursor_key, x, y);+}+fn default_cursor_session() -> String {static ID: std::sync::OnceLock<String> = std::sync::OnceLock::new();ID.get_or_init(|| format!("auto-{:x}", std::process::id()))diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rsindex 81f4aac707..c684dcdf8c 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs@@ -378,6 +378,30 @@ impl Tool for HotkeyTool {}};+ // A focus click already moved the cursor. Otherwise place a named+ // session's cursor on the element, its remembered position, or the+ // window centre, so a keyboard-first session stays visible.+ if !coordinate_focus {+ let element_center = match element_guard.clone() {+ Some(guard) => tokio::task::spawn_blocking(move || unsafe {+ crate::ax::bindings::element_screen_center(+ guard.as_ptr() as crate::ax::bindings::AXUIElementRef+ )+ })+ .await+ .ok()+ .flatten(),+ None => None,+ };+ super::cursor_tools::position_keyboard_cursor(+ &self.state,+ &args,+ window_id,+ element_center,+ )+ .await;+ }+// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──// Hotkeys like Cmd+N, Cmd+W, Cmd+T explicitly open/close// windows. The NSMenu path also briefly activates the target viadiff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rsindex 8162ec97cd..5788bad5f3 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs@@ -382,6 +382,30 @@ impl Tool for PressKeyTool {}};+ // The px form's focus click already moved the cursor. Otherwise place+ // a named session's cursor on the element, or its remembered position,+ // or the window centre, so a keyboard-first session stays visible.+ if !px_focus {+ let element_center = match pre_focus_guard.clone() {+ Some(guard) => tokio::task::spawn_blocking(move || unsafe {+ crate::ax::bindings::element_screen_center(+ guard.as_ptr() as crate::ax::bindings::AXUIElementRef+ )+ })+ .await+ .ok()+ .flatten(),+ None => None,+ };+ super::cursor_tools::position_keyboard_cursor(+ &self.state,+ &args,+ window_id,+ element_center,+ )+ .await;+ }+// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──// Single-key presses can fire autocomplete (Return on a search// box opens a results popover) or trigger menu shortcuts thatdiff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rsindex 8d7e2aa238..862588f8d2 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs@@ -356,6 +356,12 @@ impl Tool for TypeTextTool {.update_position(&cursor_key, screen_x, screen_y);}}+ // Untargeted text still gives a named session visible feedback: its+ // remembered position, or the window centre on the first action.+ if element_guard.is_none() && !used_pixel_focus {+ super::cursor_tools::position_keyboard_cursor(&self.state, &args, window_id, None)+ .await;+ }let text_clone = text.clone();let char_count = text.chars().count();diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 453d2ee5d4..4d9c80f7e4 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -29,6 +29,59 @@ fn pin_overlay_above(key: &str, hwnd: u64) {);}+/// Place and reveal a named session's cursor before keyboard or text input.+///+/// `explicit` is the targeted element's screen centre when the action has+/// one. Without it the cursor reuses its remembered position, then falls back+/// to the target window's centre, so the first keyboard action of a session+/// is visible. Anonymous calls keep their existing behavior. Geometry and+/// overlay failures are observational and never affect the input result.+async fn position_keyboard_cursor(+ state: &ToolState,+ args: &Value,+ hwnd: u64,+ explicit: Option<(f64, f64)>,+) {+ if cursor_overlay::named_session_cursor_key(args).is_none() {+ return;+ }+ let cursor_key = resolve_cursor_key(args);+ let remembered = state+ .cursor_registry+ .get(&cursor_key)+ .and_then(|cursor| cursor.x.zip(cursor.y));+ let window_center = if hwnd != 0+ && cursor_overlay::keyboard_cursor_target(explicit, remembered, None, None).is_none()+ {+ keyboard_window_center(hwnd)+ } else {+ None+ };+ let Some((x, y)) =+ cursor_overlay::keyboard_cursor_target(explicit, remembered, window_center, None)+ else {+ return;+ };+ if hwnd != 0 {+ pin_overlay_above(&cursor_key, hwnd);+ }+ overlay_glide_to(&cursor_key, x, y).await;+ state.cursor_registry.update_position(&cursor_key, x, y);+}++fn keyboard_window_center(hwnd: u64) -> Option<(f64, f64)> {+ use windows::Win32::Foundation::{HWND, RECT};+ use windows::Win32::UI::WindowsAndMessaging::GetWindowRect;+ let mut rect = RECT::default();+ unsafe { GetWindowRect(HWND(hwnd as *mut _), &mut rect) }.ok()?;+ (rect.right > rect.left && rect.bottom > rect.top).then(|| {+ (+ f64::from(rect.left + rect.right) / 2.0,+ f64::from(rect.top + rect.bottom) / 2.0,+ )+ })+}+/// Resolve the screen point a coordinate action should actuate at, scrolling/// the target element into view first when its cached center lands off-screen.///@@ -4603,6 +4656,12 @@ impl Tool for TypeTextTool {self.state.cursor_registry.update_position(&cursor_key, cx as f64, cy as f64);+ } else if args.get("x").and_then(Value::as_f64).is_none()+ || args.get("y").and_then(Value::as_f64).is_none()+ {+ // Untargeted text still gives a named session visible feedback:+ // its remembered position, or the window centre on the first action.+ position_keyboard_cursor(&self.state, &args, hwnd, None).await;}let text_len = text.chars().count();@@ -5303,6 +5362,15 @@ impl Tool for PressKeyTool {return crate::input::delivery::background_unavailable_error(hwnd, event_kind);}+ // The px form's focus click already moved the cursor. Otherwise place+ // a named session's cursor so a keyboard-first session stays visible.+ if !px_focus {+ let element_center = admitted+ .as_ref()+ .map(|element| (element.center.0 as f64, element.center.1 as f64));+ position_keyboard_cursor(&self.state, &args, hwnd, element_center).await;+ }+// W1: an element-addressed key needs the control's actual focus// target, not merely its owning top-level HWND. Embedded WebView hosts// can activate their frame from UIA SetFocus even under@@ -5702,6 +5770,15 @@ impl Tool for HotkeyTool {}};+ // The px form's focus click already moved the cursor. Otherwise place+ // a named session's cursor so a keyboard-first session stays visible.+ if !px_focus {+ let element_center = admitted+ .as_ref()+ .map(|element| (element.center.0 as f64, element.center.1 as f64));+ position_keyboard_cursor(&self.state, &args, hwnd, element_center).await;+ }+if !px_focus && crate::input::is_xaml_host_hwnd(hwnd) {let mut accelerator_keys = mods.clone();accelerator_keys.push(key.clone());@@ -8216,7 +8293,9 @@ impl Tool for SetAgentCursorThemeTool {}}-pub struct GetAgentCursorStateV2Tool;+pub struct GetAgentCursorStateV2Tool {+ state: Arc<ToolState>,+}static CURSOR_STATE_V2_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();@@ -8250,11 +8329,19 @@ impl Tool for GetAgentCursorStateV2Tool {.into_iter().flatten().collect();+ // The last point this session's cursor was placed at, `null` until+ // it first moves (same registry source as macOS).+ let position = self+ .state+ .cursor_registry+ .get(&session)+ .and_then(|cursor| cursor.x.zip(cursor.y))+ .map(|(x, y)| json!({"x": x, "y": y}));ToolResult::text(format!("Agent cursor state for session '{session}'.")).with_structured(json!({"session":session,"enabled":enabled,- "position":null,+ "position":position,"theme":{"id":theme_id,"version":version,@@ -10200,7 +10287,9 @@ pub fn build_registry_with_provider(state: state.clone(),}));r.register(Box::new(SetAgentCursorMotionV2Tool));- r.register(Box::new(GetAgentCursorStateV2Tool));+ r.register(Box::new(GetAgentCursorStateV2Tool {+ state: state.clone(),+ }));r.register(Box::new(SetAgentCursorThemeTool {state: state.clone(),}));