Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): reveal the agent cursor for keyboard-first actions (#4287)

Francesco Bonacci2d0431fb0096
diff --git a/libs/cua-driver/docs/cursor-themes.md b/libs/cua-driver/docs/cursor-themes.md
index a9b71a83a2..7b395589a7 100644
--- a/libs/cua-driver/docs/cursor-themes.md
+++ b/libs/cua-driver/docs/cursor-themes.md
@@ -74,8 +74,14 @@ cua-driver get_agent_cursor_state '{"session":"demo"}'
```
`get_agent_cursor_state` always includes `position`. It is `null` until the
-session cursor first moves, then `{"x": ..., "y": ...}` where the platform
-tracks session cursor placement.
+session cursor first moves, then `{"x": ..., "y": ...}`: the screen point the
+cursor tip was last placed on.
+
+Keyboard actions place a named session's cursor before the input.
+`press_key`, `hotkey`, and `type_text` put it on the targeted element, else
+at its last position, else at the target window's centre, so a session whose
+first action is a keyboard action shows its cursor on that window. Anonymous
+calls keep their existing cursor behavior.
`set_agent_cursor_motion` changes only movement physics and visibility timing.
It does not change artwork. The removed `set_agent_cursor_style` operation and
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs
index 7c4afdc718..00304c02d1 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs
@@ -13,12 +13,11 @@ use image::RgbaImage;
const CELL_ID: &str = "desktop-agent-cursor-showcase-px";
const SESSION: &str = "Cursor showcase";
-// MoveTo offsets the artwork centre by a 16-point vector at 45 degrees so the
-// cursor tip lands on the requested coordinate. Each axis moves by 16/sqrt(2),
-// and the session badge follows that artwork centre.
-const CURSOR_ANCHOR_OFFSET_MAGNITUDE: f64 = 16.0;
+// MoveTo anchors the artwork `POINTER_ANCHOR_OFFSET` points from the requested
+// coordinate at 45 degrees so the cursor tip lands on it. Each axis moves by
+// that offset over sqrt(2), and the session badge follows the anchor.
const CURSOR_ANCHOR_OFFSET_PER_AXIS: f64 =
- CURSOR_ANCHOR_OFFSET_MAGNITUDE * std::f64::consts::FRAC_1_SQRT_2;
+ cursor_overlay::POINTER_ANCHOR_OFFSET * std::f64::consts::FRAC_1_SQRT_2;
const POINTER_ORACLE_RADIUS: f64 = 24.0;
const BADGE_CURSOR_EXCLUSION: f64 = 34.0;
@@ -191,6 +190,262 @@ fn semantic_cursor_showcase_records_session_and_action_states() {
});
}
+const KEYBOARD_FIRST_CELL_ID: &str = "window-agent-cursor-keyboard-first-placement";
+/// Largest distance, per axis and in screen units, between the reported cursor
+/// point and the centre of the window's `list_windows` bounds. Each platform
+/// reads the centre from its own native geometry (Windows `GetWindowRect`
+/// includes the invisible resize border that the DWM frame bounds leave out),
+/// so the two can differ by a few units; a cursor left at the pointer, at a
+/// stale session point, or unplaced is far outside it.
+const WINDOW_CENTRE_TOLERANCE: f64 = 12.0;
+
+/// A named session whose first action is an untargeted `press_key` shows its
+/// agent cursor on the target window before the key is delivered: no element,
+/// no pixel target, and no remembered position leave the shared keyboard
+/// placement policy with the window centre.
+#[test]
+#[ignore]
+fn keyboard_first_session_places_its_cursor_on_the_target_window() {
+ let fixture = native_fixture();
+ let case = CaseSpec::delivered(
+ KEYBOARD_FIRST_CELL_ID,
+ fixture.toolkit,
+ fixture.toolkit,
+ "press_key_cursor_placement",
+ Targeting::NotApplicable,
+ Delivery::Foreground,
+ Scope::Window,
+ platform_route(),
+ vec![OracleKind::Protocol],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = spawn_driver_named(KEYBOARD_FIRST_CELL_ID);
+ *evidence = recording_evidence(driver.recording_dir());
+ let (pid, window_id) = launch_native_fixture(&mut driver, &fixture);
+
+ // A per-run name keeps a long-lived daemon from handing this row a
+ // cursor position that an earlier run left behind.
+ let session = format!("Keyboard-first cursor {}", std::process::id());
+ call_ok(
+ &mut driver,
+ "start_session",
+ serde_json::json!({"session": session}),
+ );
+ call_ok(
+ &mut driver,
+ "set_agent_cursor_enabled",
+ serde_json::json!({"session": session, "enabled": true}),
+ );
+ let before = agent_cursor_state(&mut driver, &session);
+ assert!(
+ before["position"].is_null(),
+ "the session cursor must start unplaced so press_key is its first placement: {before}"
+ );
+
+ driver.start_behavior_recording();
+ call_ok(
+ &mut driver,
+ "press_key",
+ serde_json::json!({
+ "session": session,
+ "pid": pid,
+ "window_id": window_id,
+ "key": "escape",
+ "delivery_mode": "foreground"
+ }),
+ );
+
+ let after = agent_cursor_state(&mut driver, &session);
+ assert_eq!(
+ after["enabled"], true,
+ "keyboard-first cursor is hidden: {after}"
+ );
+ let (Some(x), Some(y)) = (
+ after["position"]["x"].as_f64(),
+ after["position"]["y"].as_f64(),
+ ) else {
+ panic!("a keyboard-first press_key left the session cursor unplaced: {after}");
+ };
+ let (centre_x, centre_y) = window_centre(&mut driver, pid, window_id);
+ assert!(
+ (x - centre_x).abs() <= WINDOW_CENTRE_TOLERANCE
+ && (y - centre_y).abs() <= WINDOW_CENTRE_TOLERANCE,
+ "keyboard-first cursor landed at ({x:.1}, {y:.1}), expected the target window \
+ centre ({centre_x:.1}, {centre_y:.1}) within {WINDOW_CENTRE_TOLERANCE}: {after}"
+ );
+
+ call_ok(
+ &mut driver,
+ "end_session",
+ serde_json::json!({"session": session}),
+ );
+ Observation::delivered(vec![OracleKind::Protocol], Evidence::default())
+ });
+}
+
+fn agent_cursor_state(driver: &mut McpDriver, session: &str) -> serde_json::Value {
+ let response = driver.call(
+ "get_agent_cursor_state",
+ serde_json::json!({"session": session}),
+ );
+ assert!(
+ !response.is_error(),
+ "get_agent_cursor_state failed: {}",
+ response.text()
+ );
+ response.structured().clone()
+}
+
+/// Centre of the target window's `list_windows` bounds.
+fn window_centre(driver: &mut McpDriver, pid: u32, window_id: u64) -> (f64, f64) {
+ let windows = driver.call("list_windows", serde_json::json!({"pid": pid}));
+ let window = windows.structured()["windows"]
+ .as_array()
+ .and_then(|windows| {
+ windows
+ .iter()
+ .find(|window| window["window_id"].as_u64() == Some(window_id))
+ })
+ .unwrap_or_else(|| panic!("target window {window_id} is not listed: {}", windows.raw))
+ .clone();
+ let bounds = &window["bounds"];
+ let (Some(x), Some(y), Some(width), Some(height)) = (
+ bounds["x"].as_f64(),
+ bounds["y"].as_f64(),
+ bounds["width"].as_f64(),
+ bounds["height"].as_f64(),
+ ) else {
+ panic!("target window has no bounds: {window}");
+ };
+ assert!(
+ width > 0.0 && height > 0.0,
+ "target window is empty: {window}"
+ );
+ (x + width / 2.0, y + height / 2.0)
+}
+
+/// The repo-local native harness the platform's native lane stages.
+struct NativeFixture {
+ toolkit: &'static str,
+ executable: std::path::PathBuf,
+ title: &'static str,
+}
+
+#[cfg(target_os = "macos")]
+fn native_fixture() -> NativeFixture {
+ let app = std::env::var("HARNESS_APPKIT_APP")
+ .map(std::path::PathBuf::from)
+ .ok()
+ .filter(|path| path.exists())
+ .unwrap_or_else(|| {
+ cua_driver_testkit::harness_app("harness-appkit", "CuaTestHarness.AppKit.app")
+ });
+ NativeFixture {
+ toolkit: "appkit",
+ executable: app.join("Contents/MacOS/CuaTestHarness.AppKit"),
+ title: "CuaTestHarness AppKit",
+ }
+}
+
+#[cfg(target_os = "windows")]
+fn native_fixture() -> NativeFixture {
+ NativeFixture {
+ toolkit: "wpf",
+ executable: cua_driver_testkit::harness_app("harness-wpf", "CuaTestHarness.Wpf.exe"),
+ title: "CuaTestHarness WPF",
+ }
+}
+
+#[cfg(target_os = "linux")]
+fn native_fixture() -> NativeFixture {
+ NativeFixture {
+ toolkit: "gtk3",
+ executable: std::env::var("HARNESS_GTK3_EXE")
+ .map(std::path::PathBuf::from)
+ .ok()
+ .filter(|path| path.exists())
+ .unwrap_or_else(|| {
+ cua_driver_testkit::harness_app("harness-gtk3", "CuaTestHarness.Gtk3")
+ }),
+ title: "CuaTestHarness GTK3",
+ }
+}
+
+/// Launch the fixture and return the pid and id of its main window.
+fn launch_native_fixture(driver: &mut McpDriver, fixture: &NativeFixture) -> (u32, u64) {
+ assert!(
+ fixture.executable.exists(),
+ "required {} harness is missing at {:?}; run the fixture build",
+ fixture.toolkit,
+ fixture.executable
+ );
+ let listed_window_ids = |driver: &mut McpDriver| -> std::collections::HashSet<u64> {
+ driver
+ .call("list_windows", serde_json::json!({}))
+ .structured()["windows"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .filter_map(|window| window["window_id"].as_u64())
+ .collect()
+ };
+ let earlier_windows = listed_window_ids(driver);
+ let child = cua_driver_testkit::spawn_in_job(
+ std::process::Command::new(&fixture.executable)
+ .stdout(std::process::Stdio::null())
+ .stderr(std::process::Stdio::null()),
+ )
+ .unwrap_or_else(|error| panic!("launch {} harness: {error}", fixture.toolkit));
+ driver.reaper().push(child);
+
+ // Take the new window with the fixture title, whatever process owns it:
+ // a toolkit launcher can hand the window to another process, and an
+ // earlier fixture window must not be mistaken for this one.
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ loop {
+ let windows = driver.call("list_windows", serde_json::json!({}));
+ let found = windows.structured()["windows"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .find(|window| {
+ window["title"]
+ .as_str()
+ .is_some_and(|title| title.contains(fixture.title))
+ && window["window_id"]
+ .as_u64()
+ .is_some_and(|id| !earlier_windows.contains(&id))
+ })
+ .and_then(|window| {
+ Some((
+ u32::try_from(window["pid"].as_u64()?).ok()?,
+ window["window_id"].as_u64()?,
+ ))
+ });
+ if let Some((pid, window_id)) =
+ found.filter(|(pid, window_id)| *pid != 0 && *window_id != 0)
+ {
+ driver.reaper().track_pid(pid);
+ #[cfg(target_os = "macos")]
+ while !cua_driver_testkit::observer::macos::application_presented(pid) {
+ assert!(
+ std::time::Instant::now() < deadline,
+ "AppKit harness pid {pid} did not finish launching"
+ );
+ settle(50);
+ }
+ settle(500);
+ return (pid, window_id);
+ }
+ assert!(
+ std::time::Instant::now() < deadline,
+ "{} harness window never appeared",
+ fixture.toolkit
+ );
+ settle(250);
+ }
+}
+
fn assert_cursor_and_badge_pixels_changed(
baseline: &RgbaImage,
cursor_frame: &RgbaImage,
@@ -791,14 +1046,18 @@ mod pixel_oracle_tests {
}
}
-#[cfg(target_os = "macos")]
fn spawn_driver() -> McpDriver {
- McpDriver::spawn_macos_daemon_proxy_named(CELL_ID).expect("start installed macOS daemon proxy")
+ spawn_driver_named(CELL_ID)
+}
+
+#[cfg(target_os = "macos")]
+fn spawn_driver_named(cell_id: &str) -> McpDriver {
+ McpDriver::spawn_macos_daemon_proxy_named(cell_id).expect("start installed macOS daemon proxy")
}
#[cfg(not(target_os = "macos"))]
-fn spawn_driver() -> McpDriver {
- McpDriver::spawn_named_with_overlay(CELL_ID)
+fn spawn_driver_named(cell_id: &str) -> McpDriver {
+ McpDriver::spawn_named_with_overlay(cell_id)
.expect("start source-built driver with native cursor overlay")
}
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs
index 36f37172d2..ce000fd313 100644
--- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs
@@ -408,6 +408,35 @@ pub fn pointer_for_anchor(x: f64, y: f64, heading: f64) -> (f64, f64) {
)
}
+/// Cursor key for a named session's keyboard and text feedback: the explicit
+/// `session` label, else the trusted lifecycle `_session_id`. Anonymous calls
+/// return `None` and keep their existing behavior.
+pub fn named_session_cursor_key(args: &serde_json::Value) -> Option<String> {
+ ["session", "_session_id"].into_iter().find_map(|key| {
+ args.get(key)
+ .and_then(serde_json::Value::as_str)
+ .filter(|session| !session.is_empty())
+ .map(str::to_owned)
+ })
+}
+
+/// Where a keyboard or text action places its session cursor: the explicit
+/// element or pixel target, else the cursor's remembered position, else the
+/// target window's centre, else the current pointer. Non-finite points are
+/// skipped so a bad geometry read cannot poison position reuse.
+pub fn keyboard_cursor_target(
+ explicit: Option<(f64, f64)>,
+ remembered: Option<(f64, f64)>,
+ window_center: Option<(f64, f64)>,
+ current_pointer: Option<(f64, f64)>,
+) -> Option<(f64, f64)> {
+ let finite = |point: Option<(f64, f64)>| point.filter(|(x, y)| x.is_finite() && y.is_finite());
+ finite(explicit)
+ .or_else(|| finite(remembered))
+ .or_else(|| finite(window_center))
+ .or_else(|| finite(current_pointer))
+}
+
/// Build the shared overlay command for one native pointer position.
///
/// Native drag implementations report the actual event coordinate. Anchoring
@@ -470,6 +499,48 @@ mod pointer_tracking_tests {
assert!(!sibling.get());
}
+ #[test]
+ fn named_sessions_opt_into_keyboard_cursor_positioning() {
+ use serde_json::json;
+ assert_eq!(
+ named_session_cursor_key(&json!({"session": "editing-run"})).as_deref(),
+ Some("editing-run")
+ );
+ assert_eq!(
+ named_session_cursor_key(&json!({"_session_id": "implicit"})).as_deref(),
+ Some("implicit")
+ );
+ assert_eq!(
+ named_session_cursor_key(&json!({"cursor_id": "legacy"})),
+ None
+ );
+ assert_eq!(named_session_cursor_key(&json!({"session": ""})), None);
+ }
+
+ #[test]
+ fn keyboard_cursor_uses_explicit_then_remembered_then_window_then_pointer() {
+ let explicit = Some((10.0, 20.0));
+ let remembered = Some((30.0, 40.0));
+ let window = Some((50.0, 60.0));
+ let pointer = Some((70.0, 80.0));
+ assert_eq!(
+ keyboard_cursor_target(explicit, remembered, window, pointer),
+ explicit
+ );
+ assert_eq!(
+ keyboard_cursor_target(None, remembered, window, pointer),
+ remembered
+ );
+ assert_eq!(keyboard_cursor_target(None, None, window, pointer), window);
+ assert_eq!(keyboard_cursor_target(None, None, None, pointer), pointer);
+ assert_eq!(keyboard_cursor_target(None, None, None, None), None);
+ assert_eq!(
+ keyboard_cursor_target(Some((f64::NAN, 1.0)), Some((12.0, 34.0)), None, None),
+ Some((12.0, 34.0)),
+ "invalid coordinates must not poison session position reuse"
+ );
+ }
+
#[test]
fn session_cleanup_removes_named_cursor_but_preserves_anonymous_default() {
let registry = CursorRegistry::new();
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs
index 264c427615..293491bc0d 100644
--- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs
+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs
@@ -596,7 +596,10 @@ impl RenderStateCore {
}
let idle_hide_ms = self.motion.idle_hide_ms;
if idle_hide_ms > 0.0 {
- let moving = self.path.is_some() || self.spring.is_some() || self.click_t.is_some();
+ let moving = self.path.is_some()
+ || self.spring.is_some()
+ || self.click_t.is_some()
+ || self.visual.resolved_action != CursorAction::Idle;
if moving {
self.idle_secs = 0.0;
self.idle_alpha = 1.0;
@@ -768,6 +771,10 @@ impl RenderStateCore {
target,
} => {
self.visual.begin(action, delivery, target);
+ // A semantic cue re-reveals an idle-faded cursor that already
+ // has a position; `tick_idle` keeps it visible until the cue ends.
+ self.idle_secs = 0.0;
+ self.idle_alpha = 1.0;
self.badge_modifiers = if delivery.is_some() || target.is_some() {
Some((delivery, target))
} else {
@@ -1094,6 +1101,43 @@ mod session_badge_and_action_tests {
assert_eq!(core.idle_alpha, 1.0);
}
+ #[test]
+ fn semantic_cue_re_reveals_an_idle_hidden_cursor_until_it_ends() {
+ let frame = 1.0 / 60.0;
+ let mut core = RenderStateCore::new(CursorConfig::default());
+ core.motion.idle_hide_ms = 200.0;
+ core.apply_command_base(
+ OverlayCommand::ClickPulse { x: 40.0, y: 60.0 },
+ false,
+ false,
+ );
+ for _ in 0..120 {
+ core.tick_motion(frame);
+ }
+ assert_eq!(core.idle_alpha, 0.0, "the positioned cursor idles out");
+
+ core.apply_command_base(
+ OverlayCommand::BeginAction {
+ action: CursorAction::Text,
+ delivery: None,
+ target: None,
+ },
+ false,
+ false,
+ );
+ assert_eq!(core.idle_alpha, 1.0, "a keyboard cue re-reveals it");
+ for _ in 0..120 {
+ core.tick_motion(frame);
+ }
+ assert_eq!(core.idle_alpha, 1.0, "it stays visible while the cue runs");
+
+ core.apply_command_base(OverlayCommand::EndAction(CursorAction::Text), false, false);
+ for _ in 0..120 {
+ core.tick_motion(frame);
+ }
+ assert_eq!(core.idle_alpha, 0.0, "it idles out again after the cue");
+ }
+
#[test]
fn session_badge_holds_then_fades_once() {
let mut core = RenderStateCore::new(CursorConfig::default());
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index c2017c7afb..5858fb1edb 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -5132,28 +5132,7 @@ fn resolve_cursor_key(args: &Value) -> String {
/// `_session_id` is trusted lifecycle state and must behave like a public named
/// session for cursor ownership.
fn named_session_cursor_key(args: &Value) -> Option<String> {
- ["session", "_session_id"].into_iter().find_map(|key| {
- args.get(key)
- .and_then(Value::as_str)
- .filter(|session| !session.is_empty())
- .map(str::to_owned)
- })
-}
-
-fn finite_cursor_point(point: Option<(f64, f64)>) -> Option<(f64, f64)> {
- point.filter(|(x, y)| x.is_finite() && y.is_finite())
-}
-
-fn choose_keyboard_cursor_target(
- explicit: Option<(f64, f64)>,
- remembered: Option<(f64, f64)>,
- window_center: Option<(f64, f64)>,
- current_pointer: Option<(f64, f64)>,
-) -> Option<(f64, f64)> {
- finite_cursor_point(explicit)
- .or_else(|| finite_cursor_point(remembered))
- .or_else(|| finite_cursor_point(window_center))
- .or_else(|| finite_cursor_point(current_pointer))
+ cursor_overlay::named_session_cursor_key(args)
}
fn mouse_hold_json(cursor_id: &str, hold: Option<&MouseHoldState>) -> Value {
@@ -5428,7 +5407,7 @@ async fn position_named_session_keyboard_cursor(
let fallback = if named_cursor_id.is_some()
&& explicit.is_none()
- && finite_cursor_point(remembered).is_none()
+ && cursor_overlay::keyboard_cursor_target(None, remembered, None, None).is_none()
{
tokio::task::spawn_blocking(move || {
let center = keyboard_window_center(xid);
@@ -5442,7 +5421,7 @@ async fn position_named_session_keyboard_cursor(
};
let Some((sx, sy)) =
- choose_keyboard_cursor_target(explicit, remembered, fallback.0, fallback.1)
+ cursor_overlay::keyboard_cursor_target(explicit, remembered, fallback.0, fallback.1)
else {
return;
};
@@ -12595,7 +12574,9 @@ impl Tool for SetAgentCursorThemeTool {
}
}
-pub struct GetAgentCursorStateV2Tool;
+pub struct GetAgentCursorStateV2Tool {
+ state: Arc<ToolState>,
+}
static CURSOR_STATE_V2_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
@@ -12626,11 +12607,19 @@ impl Tool for GetAgentCursorStateV2Tool {
.into_iter()
.flatten()
.collect();
+ // The last point this session's cursor was placed at, `null` until
+ // it first moves (same registry source as macOS).
+ let position = self
+ .state
+ .cursor_registry
+ .get(&session)
+ .and_then(|cursor| cursor.x.zip(cursor.y))
+ .map(|(x, y)| json!({"x": x, "y": y}));
ToolResult::text(format!("Agent cursor state for session '{session}'.")).with_structured(
json!({
"session":session,
"enabled":enabled,
- "position":null,
+ "position":position,
"theme":{
"id":theme_id,
"version":version,
@@ -14148,7 +14137,9 @@ pub fn build_registry_with_provider(
state: state.clone(),
}));
r.register(Box::new(SetAgentCursorMotionV2Tool));
- r.register(Box::new(GetAgentCursorStateV2Tool));
+ r.register(Box::new(GetAgentCursorStateV2Tool {
+ state: state.clone(),
+ }));
r.register(Box::new(SetAgentCursorThemeTool {
state: state.clone(),
}));
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/session_cursor_target_tests.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/session_cursor_target_tests.rs
index 7860153cc5..d5ee4b44a9 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/session_cursor_target_tests.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/session_cursor_target_tests.rs
@@ -1,59 +1,6 @@
-use super::{
- choose_keyboard_cursor_target, cursor_control_scope, named_session_cursor_key,
- reveal_pointer_action_for, CursorControlScope, ToolState,
-};
+use super::{cursor_control_scope, reveal_pointer_action_for, CursorControlScope, ToolState};
use serde_json::json;
-#[test]
-fn lifecycle_owned_sessions_opt_into_keyboard_cursor_positioning() {
- assert_eq!(
- named_session_cursor_key(&json!({"session": "editing-run"})).as_deref(),
- Some("editing-run")
- );
- assert_eq!(
- named_session_cursor_key(&json!({"cursor_id": "legacy"})),
- None
- );
- assert_eq!(
- named_session_cursor_key(&json!({"_session_id": "implicit"})).as_deref(),
- Some("implicit")
- );
- assert_eq!(named_session_cursor_key(&json!({})), None);
-}
-
-#[test]
-fn keyboard_cursor_uses_explicit_then_remembered_then_safe_seed() {
- let explicit = Some((10.0, 20.0));
- let remembered = Some((30.0, 40.0));
- let window_center = Some((50.0, 60.0));
- let current_pointer = Some((70.0, 80.0));
-
- assert_eq!(
- choose_keyboard_cursor_target(explicit, remembered, window_center, current_pointer),
- explicit
- );
- assert_eq!(
- choose_keyboard_cursor_target(None, remembered, window_center, current_pointer),
- remembered
- );
- assert_eq!(
- choose_keyboard_cursor_target(None, None, window_center, current_pointer),
- window_center
- );
- assert_eq!(
- choose_keyboard_cursor_target(None, None, None, current_pointer),
- current_pointer
- );
-}
-
-#[test]
-fn invalid_coordinates_do_not_poison_session_position_reuse() {
- assert_eq!(
- choose_keyboard_cursor_target(Some((f64::NAN, 1.0)), Some((12.0, 34.0)), None, None,),
- Some((12.0, 34.0))
- );
-}
-
#[test]
fn real_pointer_control_requires_explicit_desktop_scope() {
assert_eq!(cursor_control_scope(&json!({})), CursorControlScope::Agent);
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs
index 8023a2055b..09e89171c7 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs
@@ -24,6 +24,59 @@ pub(crate) fn resolve_cursor_key(args: &Value) -> String {
default_cursor_session()
}
+/// Place and reveal a named session's cursor before keyboard or text input.
+///
+/// `explicit` is the targeted element's screen centre when the action has
+/// one. Without it the cursor reuses its remembered position, then falls back
+/// to the target window's centre, so the first keyboard action of a session
+/// is visible. Anonymous calls keep their existing behavior. Geometry and
+/// overlay failures are observational and never affect the input result.
+pub(crate) async fn position_keyboard_cursor(
+ state: &ToolState,
+ args: &Value,
+ window_id: Option<u32>,
+ explicit: Option<(f64, f64)>,
+) {
+ let Some(cursor_key) = cursor_overlay::named_session_cursor_key(args) else {
+ return;
+ };
+ let remembered = state
+ .cursor_registry
+ .get(&cursor_key)
+ .and_then(|cursor| cursor.position.map(|position| (position.x, position.y)));
+ let window_center = match window_id {
+ Some(wid)
+ if cursor_overlay::keyboard_cursor_target(explicit, remembered, None, None)
+ .is_none() =>
+ {
+ tokio::task::spawn_blocking(move || {
+ let bounds = crate::windows::window_bounds_by_id(wid)?;
+ (bounds.width > 0.0 && bounds.height > 0.0).then_some((
+ bounds.x + bounds.width / 2.0,
+ bounds.y + bounds.height / 2.0,
+ ))
+ })
+ .await
+ .ok()
+ .flatten()
+ }
+ _ => None,
+ };
+ let Some((x, y)) =
+ cursor_overlay::keyboard_cursor_target(explicit, remembered, window_center, None)
+ else {
+ return;
+ };
+ if let Some(wid) = window_id {
+ crate::cursor::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::PinAbove(u64::from(wid)),
+ );
+ }
+ crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), x, y).await;
+ state.cursor_registry.update_position(&cursor_key, x, y);
+}
+
fn default_cursor_session() -> String {
static ID: std::sync::OnceLock<String> = std::sync::OnceLock::new();
ID.get_or_init(|| format!("auto-{:x}", std::process::id()))
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
index 81f4aac707..c684dcdf8c 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
@@ -378,6 +378,30 @@ impl Tool for HotkeyTool {
}
};
+ // A focus click already moved the cursor. Otherwise place a named
+ // session's cursor on the element, its remembered position, or the
+ // window centre, so a keyboard-first session stays visible.
+ if !coordinate_focus {
+ let element_center = match element_guard.clone() {
+ Some(guard) => tokio::task::spawn_blocking(move || unsafe {
+ crate::ax::bindings::element_screen_center(
+ guard.as_ptr() as crate::ax::bindings::AXUIElementRef
+ )
+ })
+ .await
+ .ok()
+ .flatten(),
+ None => None,
+ };
+ super::cursor_tools::position_keyboard_cursor(
+ &self.state,
+ &args,
+ window_id,
+ element_center,
+ )
+ .await;
+ }
+
// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──
// Hotkeys like Cmd+N, Cmd+W, Cmd+T explicitly open/close
// windows. The NSMenu path also briefly activates the target via
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
index 8162ec97cd..5788bad5f3 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
@@ -382,6 +382,30 @@ impl Tool for PressKeyTool {
}
};
+ // The px form's focus click already moved the cursor. Otherwise place
+ // a named session's cursor on the element, or its remembered position,
+ // or the window centre, so a keyboard-first session stays visible.
+ if !px_focus {
+ let element_center = match pre_focus_guard.clone() {
+ Some(guard) => tokio::task::spawn_blocking(move || unsafe {
+ crate::ax::bindings::element_screen_center(
+ guard.as_ptr() as crate::ax::bindings::AXUIElementRef
+ )
+ })
+ .await
+ .ok()
+ .flatten(),
+ None => None,
+ };
+ super::cursor_tools::position_keyboard_cursor(
+ &self.state,
+ &args,
+ window_id,
+ element_center,
+ )
+ .await;
+ }
+
// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──
// Single-key presses can fire autocomplete (Return on a search
// box opens a results popover) or trigger menu shortcuts that
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
index 8d7e2aa238..862588f8d2 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
@@ -356,6 +356,12 @@ impl Tool for TypeTextTool {
.update_position(&cursor_key, screen_x, screen_y);
}
}
+ // Untargeted text still gives a named session visible feedback: its
+ // remembered position, or the window centre on the first action.
+ if element_guard.is_none() && !used_pixel_focus {
+ super::cursor_tools::position_keyboard_cursor(&self.state, &args, window_id, None)
+ .await;
+ }
let text_clone = text.clone();
let char_count = text.chars().count();
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 453d2ee5d4..4d9c80f7e4 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -29,6 +29,59 @@ fn pin_overlay_above(key: &str, hwnd: u64) {
);
}
+/// Place and reveal a named session's cursor before keyboard or text input.
+///
+/// `explicit` is the targeted element's screen centre when the action has
+/// one. Without it the cursor reuses its remembered position, then falls back
+/// to the target window's centre, so the first keyboard action of a session
+/// is visible. Anonymous calls keep their existing behavior. Geometry and
+/// overlay failures are observational and never affect the input result.
+async fn position_keyboard_cursor(
+ state: &ToolState,
+ args: &Value,
+ hwnd: u64,
+ explicit: Option<(f64, f64)>,
+) {
+ if cursor_overlay::named_session_cursor_key(args).is_none() {
+ return;
+ }
+ let cursor_key = resolve_cursor_key(args);
+ let remembered = state
+ .cursor_registry
+ .get(&cursor_key)
+ .and_then(|cursor| cursor.x.zip(cursor.y));
+ let window_center = if hwnd != 0
+ && cursor_overlay::keyboard_cursor_target(explicit, remembered, None, None).is_none()
+ {
+ keyboard_window_center(hwnd)
+ } else {
+ None
+ };
+ let Some((x, y)) =
+ cursor_overlay::keyboard_cursor_target(explicit, remembered, window_center, None)
+ else {
+ return;
+ };
+ if hwnd != 0 {
+ pin_overlay_above(&cursor_key, hwnd);
+ }
+ overlay_glide_to(&cursor_key, x, y).await;
+ state.cursor_registry.update_position(&cursor_key, x, y);
+}
+
+fn keyboard_window_center(hwnd: u64) -> Option<(f64, f64)> {
+ use windows::Win32::Foundation::{HWND, RECT};
+ use windows::Win32::UI::WindowsAndMessaging::GetWindowRect;
+ let mut rect = RECT::default();
+ unsafe { GetWindowRect(HWND(hwnd as *mut _), &mut rect) }.ok()?;
+ (rect.right > rect.left && rect.bottom > rect.top).then(|| {
+ (
+ f64::from(rect.left + rect.right) / 2.0,
+ f64::from(rect.top + rect.bottom) / 2.0,
+ )
+ })
+}
+
/// Resolve the screen point a coordinate action should actuate at, scrolling
/// the target element into view first when its cached center lands off-screen.
///
@@ -4603,6 +4656,12 @@ impl Tool for TypeTextTool {
self.state
.cursor_registry
.update_position(&cursor_key, cx as f64, cy as f64);
+ } else if args.get("x").and_then(Value::as_f64).is_none()
+ || args.get("y").and_then(Value::as_f64).is_none()
+ {
+ // Untargeted text still gives a named session visible feedback:
+ // its remembered position, or the window centre on the first action.
+ position_keyboard_cursor(&self.state, &args, hwnd, None).await;
}
let text_len = text.chars().count();
@@ -5303,6 +5362,15 @@ impl Tool for PressKeyTool {
return crate::input::delivery::background_unavailable_error(hwnd, event_kind);
}
+ // The px form's focus click already moved the cursor. Otherwise place
+ // a named session's cursor so a keyboard-first session stays visible.
+ if !px_focus {
+ let element_center = admitted
+ .as_ref()
+ .map(|element| (element.center.0 as f64, element.center.1 as f64));
+ position_keyboard_cursor(&self.state, &args, hwnd, element_center).await;
+ }
+
// W1: an element-addressed key needs the control's actual focus
// target, not merely its owning top-level HWND. Embedded WebView hosts
// can activate their frame from UIA SetFocus even under
@@ -5702,6 +5770,15 @@ impl Tool for HotkeyTool {
}
};
+ // The px form's focus click already moved the cursor. Otherwise place
+ // a named session's cursor so a keyboard-first session stays visible.
+ if !px_focus {
+ let element_center = admitted
+ .as_ref()
+ .map(|element| (element.center.0 as f64, element.center.1 as f64));
+ position_keyboard_cursor(&self.state, &args, hwnd, element_center).await;
+ }
+
if !px_focus && crate::input::is_xaml_host_hwnd(hwnd) {
let mut accelerator_keys = mods.clone();
accelerator_keys.push(key.clone());
@@ -8216,7 +8293,9 @@ impl Tool for SetAgentCursorThemeTool {
}
}
-pub struct GetAgentCursorStateV2Tool;
+pub struct GetAgentCursorStateV2Tool {
+ state: Arc<ToolState>,
+}
static CURSOR_STATE_V2_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
@@ -8250,11 +8329,19 @@ impl Tool for GetAgentCursorStateV2Tool {
.into_iter()
.flatten()
.collect();
+ // The last point this session's cursor was placed at, `null` until
+ // it first moves (same registry source as macOS).
+ let position = self
+ .state
+ .cursor_registry
+ .get(&session)
+ .and_then(|cursor| cursor.x.zip(cursor.y))
+ .map(|(x, y)| json!({"x": x, "y": y}));
ToolResult::text(format!("Agent cursor state for session '{session}'.")).with_structured(
json!({
"session":session,
"enabled":enabled,
- "position":null,
+ "position":position,
"theme":{
"id":theme_id,
"version":version,
@@ -10200,7 +10287,9 @@ pub fn build_registry_with_provider(
state: state.clone(),
}));
r.register(Box::new(SetAgentCursorMotionV2Tool));
- r.register(Box::new(GetAgentCursorStateV2Tool));
+ r.register(Box::new(GetAgentCursorStateV2Tool {
+ state: state.clone(),
+ }));
r.register(Box::new(SetAgentCursorThemeTool {
state: state.clone(),
}));