Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): say when a tooltip needs the app under the pointer active (#4865)
Francesco Bonacci7dbead00a2ac
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/move_cursor.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/move_cursor.rsindex 07128d13ed..e52ddbccdc 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/move_cursor.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/move_cursor.rs@@ -65,9 +65,15 @@ impl Tool for MoveCursorTool {let result =tokio::task::spawn_blocking(move || crate::input::mouse::move_cursor_desktop(x, y)).await;+ let note = tokio::task::spawn_blocking(move || inactive_app_under_point(x, y))+ .await+ .ok()+ .flatten()+ .map(|app| inactive_app_note(&app))+ .unwrap_or_default();return match result {Ok(Ok(())) => ToolResult::text(format!(- "Moved the real desktop pointer to ({x:.1}, {y:.1})."+ "Moved the real desktop pointer to ({x:.1}, {y:.1}).{note}")).with_structured(serde_json::json!({"scope": "desktop",@@ -108,3 +114,39 @@ impl Tool for MoveCursorTool {))}}++/// The app that owns the frontmost ordinary window under a desktop point,+/// when that app is not the active one. The driver's own windows (cursor+/// overlay) are skipped.+fn inactive_app_under_point(x: f64, y: f64) -> Option<String> {+ let own = std::process::id() as i32;+ let under = crate::windows::visible_windows()+ .into_iter()+ .filter(|w| w.layer == 0 && w.is_on_screen && w.pid != own)+ .filter(|w| {+ let b = &w.bounds;+ x >= b.x && x < b.x + b.width && y >= b.y && y < b.y + b.height+ })+ .max_by_key(|w| w.z_index)?;+ (crate::apps::frontmost_pid() != Some(under.pid)).then_some(under.app_name)+}++/// Hover tracking reaches an inactive app, but AppKit shows native tooltips+/// only in the active one (VM check: 3/3 with the app active, 0/3 without).+fn inactive_app_note(app: &str) -> String {+ format!(+ " {app} is not the active app: hover effects update, but native tooltips only \+ appear in the active app. To read a tooltip, bring_to_front its window first, \+ move the pointer onto the control and wait about a second before looking."+ )+}++#[cfg(test)]+mod tests {+ #[test]+ fn the_inactive_app_note_names_the_route_to_a_tooltip() {+ let note = super::inactive_app_note("BenchLab");+ assert!(note.starts_with(" BenchLab is not the active app"));+ assert!(note.contains("bring_to_front"));+ }+}