Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): share overlay query reads across Linux and Windows (#6117)
injaneity6097f4938fcd
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rsindex bfb5fb0f93..123050e0f5 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs@@ -34,8 +34,8 @@ pub use motion::{};pub use path_planner::{PathPlanner, PathState, PlannedPath};pub use render_map::{- keyed_config, seed_position, CursorMap, MsgOutcome, RenderEntry, RenderMap, ScreenFrame,- DEFAULT_CURSOR_KEY, SEED_OFFSET,+ keyed_config, seed_position, CursorMap, CursorThemeState, MsgOutcome, RenderEntry, RenderMap,+ ScreenFrame, DEFAULT_CURSOR_KEY, SEED_OFFSET,};pub use render_state::{paint_cursor, render_frame, FocusRect, RenderStateCore, SESSION_BADGE_FADE_SECS,diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rsindex 5f7d29bfb0..de375d41f7 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs@@ -28,7 +28,8 @@ use std::time::Duration;use indexmap::IndexMap;use crate::{- CursorConfig, CursorKey, KeyedOverlayCommand, OverlayCommand, OverlayMsg, RenderStateCore,+ CursorConfig, CursorKey, CursorVisualState, KeyedOverlayCommand, MotionConfig, OverlayCommand,+ OverlayMsg, RenderStateCore,};/// Key of the anonymous cursor seeded at startup. It backs one-shot calls@@ -39,6 +40,10 @@ pub const DEFAULT_CURSOR_KEY: &str = "default";/// 140 points reads as motion at the reference peak glide speed.pub const SEED_OFFSET: f64 = 140.0;+/// A cursor's painted theme and semantic playback state: theme id, version,+/// profile, non-fatal fallback reason, and visual state.+pub type CursorThemeState = (String, String, String, Option<String>, CursorVisualState);+/// Insertion-ordered cursor collection. Iteration order is the paint order.pub type CursorMap<S> = IndexMap<CursorKey, S>;@@ -262,6 +267,38 @@ impl<S: RenderEntry, P> RenderMap<S, P> {.or_else(|| self.cursors.get(DEFAULT_CURSOR_KEY))}+ /// Whether `key`'s cursor (or the `"default"` cursor it falls back to)+ /// is user-visible.+ pub fn is_enabled(&self, key: &str) -> bool {+ self.cursor_or_default(key)+ .is_some_and(|cursor| cursor.core().visible)+ }++ /// `key`'s own on-screen position. `None` for an absent or unplaced+ /// cursor; never the `"default"` cursor's position.+ pub fn placed_position(&self, key: &str) -> Option<(f64, f64)> {+ self.cursors+ .get(key)+ .map(S::core)+ .filter(|core| core.placed)+ .map(|core| core.pos)+ }++ /// `key`'s motion config, falling back to the `"default"` cursor's.+ pub fn motion(&self, key: &str) -> MotionConfig {+ self.cursor_or_default(key)+ .map(|cursor| cursor.core().motion.clone())+ .unwrap_or_default()+ }++ /// The theme and visual state `key`'s cursor (or the `"default"`+ /// cursor) is painting.+ pub fn theme_state(&self, key: &str) -> Option<CursorThemeState> {+ let core = self.cursor_or_default(key)?.core();+ let (id, version, profile, fallback) = core.active_theme_metadata();+ Some((id, version, profile, fallback, core.visual.clone()))+ }+/// Apply one drained channel message.pub fn apply_msg(&mut self, msg: OverlayMsg) -> MsgOutcome {match msg {@@ -806,4 +843,95 @@ mod tests {}assert_eq!(arrivals, ["sessA"]);}++ #[test]+ fn queries_fall_back_to_the_default_cursor_until_a_session_owns_one() {+ let mut map = map();+ let default_motion = map.cursors[DEFAULT_CURSOR_KEY].motion.clone();+ assert_eq!(default_motion.idle_hide_ms, 500.0);+ assert!(map.is_enabled("s"));+ assert_eq!(map.motion("s"), default_motion);+ let default = &mut map.cursors[DEFAULT_CURSOR_KEY];+ default.theme = Some(test_theme("test.default", "1.0.0", "test-default"));+ default.theme_fallback = Some("installed theme unreadable".into());+ default.visual.requested_action = crate::CursorAction::Observe;+ let default_visual = default.visual.clone();+ assert_eq!(+ map.theme_state("s"),+ Some((+ "test.default".to_owned(),+ "1.0.0".to_owned(),+ "test-default".to_owned(),+ Some("installed theme unreadable".to_owned()),+ default_visual.clone(),+ ))+ );++ map.cursors[DEFAULT_CURSOR_KEY].visible = false;+ assert!(+ !map.is_enabled("s"),+ "hiding the default hides an unowned session"+ );++ let own = map.cursor_mut("s").unwrap();+ own.motion.idle_hide_ms = 900.0;+ assert!(map.is_enabled("s"), "a session's own cursor answers for it");+ assert_eq!(map.motion("s").idle_hide_ms, 900.0);+ assert_eq!(map.motion(DEFAULT_CURSOR_KEY).idle_hide_ms, 500.0);++ let own = map.cursor_mut("s").unwrap();+ own.theme = Some(test_theme("test.session", "2.0.0", "test-session"));+ own.theme_fallback = None;+ own.visual.requested_action = crate::CursorAction::Click;+ let own_visual = own.visual.clone();+ assert_ne!(own_visual, default_visual);+ assert_eq!(+ map.theme_state("s"),+ Some((+ "test.session".to_owned(),+ "2.0.0".to_owned(),+ "test-session".to_owned(),+ None,+ own_visual,+ ))+ );+ assert_eq!(+ map.theme_state(DEFAULT_CURSOR_KEY).map(|state| state.0),+ Some("test.default".to_owned())+ );+ }++ fn test_theme(id: &str, version: &str, profile: &str) -> std::sync::Arc<crate::CompiledTheme> {+ std::sync::Arc::new(crate::CompiledTheme {+ id: id.to_owned(),+ name: id.to_owned(),+ version: version.to_owned(),+ author: String::new(),+ license: String::new(),+ profile: profile.to_owned(),+ source_hash: [0; 32],+ hotspot: [0, 0],+ actions: Default::default(),+ })+ }++ #[test]+ fn placed_position_is_exact_key_and_requires_placement() {+ let mut map = map();+ placed(&mut map, DEFAULT_CURSOR_KEY);+ assert_eq!(+ map.placed_position(DEFAULT_CURSOR_KEY),+ Some((100.0, 100.0))+ );+ assert_eq!(map.placed_position("s"), None, "never inherits the default");++ map.cursor_mut("s").unwrap();+ assert_eq!(map.placed_position("s"), None, "created but unplaced");++ placed(&mut map, "s");+ assert_eq!(map.placed_position("s"), Some((100.0, 100.0)));++ map.remove("s".to_owned());+ assert_eq!(map.placed_position("s"), None);+ }}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rsindex d0e4356105..7065c444c4 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs@@ -610,15 +610,13 @@ fn dispatch_shell_helper_command(key: &str, cmd: &OverlayCommand, semantic: bool}}+/// Read the render map, treating a poisoned or uninitialised map as absent.+fn read_render<T>(f: impl FnOnce(&RenderMap) -> T) -> Option<T> {+ RENDER.lock().ok()?.as_ref().map(f)+}+pub fn is_enabled_for(key: &str) -> bool {- RENDER- .lock()- .ok()- .and_then(|g| {- g.as_ref()- .and_then(|m| m.cursor_or_default(key).map(|rs| rs.core.visible))- })- .unwrap_or(false)+ read_render(|map| map.is_enabled(key)).unwrap_or(false)}/// Truthful render acknowledgement for lifecycle inspection. This checks the@@ -637,41 +635,15 @@ pub fn is_visible_for_session(key: &str) -> bool {}pub fn current_position_for(key: &str) -> Option<(f64, f64)> {- RENDER.lock().ok().and_then(|g| {- g.as_ref()- .and_then(|m| m.cursors.get(key))- .filter(|rs| rs.core.placed)- .map(|rs| rs.core.pos)- })+ read_render(|map| map.placed_position(key)).flatten()}pub fn current_motion_for(key: &str) -> cursor_overlay::MotionConfig {- RENDER- .lock()- .ok()- .and_then(|guard| {- guard.as_ref().and_then(|map| {- map.cursor_or_default(key)- .map(|state| state.core.motion.clone())- })- })- .unwrap_or_default()+ read_render(|map| map.motion(key)).unwrap_or_default()}-pub fn current_theme_state_for(- key: &str,-) -> Option<(- String,- String,- String,- Option<String>,- cursor_overlay::CursorVisualState,-)> {- let guard = RENDER.lock().ok()?;- let map = guard.as_ref()?;- let state = map.cursor_or_default(key)?;- let (id, version, profile, fallback) = state.core.active_theme_metadata();- Some((id, version, profile, fallback, state.core.visual.clone()))+pub fn current_theme_state_for(key: &str) -> Option<cursor_overlay::CursorThemeState> {+ read_render(|map| map.theme_state(key)).flatten()}fn seed_start_if_unplaced(key: &CursorKey, target_x: f64, target_y: f64) -> bool {diff --git a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rsindex e9c677e83c..6ee87678cf 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs@@ -302,17 +302,15 @@ pub fn revive_cursor(key: CursorKey) {}}+/// Read the render map, treating a poisoned or uninitialised map as absent.+fn read_render<T>(f: impl FnOnce(&RenderMap) -> T) -> Option<T> {+ RENDER.lock().ok()?.as_ref().map(f)+}+/// Returns true if the cursor for `key` is currently enabled/visible. A session/// with no own cursor yet falls back to the seeded `"default"` cursor.pub fn is_enabled(key: &str) -> bool {- RENDER- .lock()- .ok()- .and_then(|g| {- g.as_ref()- .and_then(|m| m.cursor_or_default(key).map(|rs| rs.core.visible))- })- .unwrap_or(false)+ read_render(|map| map.is_enabled(key)).unwrap_or(false)}/// Truthful render acknowledgement for lifecycle inspection. Unlike@@ -334,40 +332,16 @@ pub fn is_visible_for_session(key: &str) -> bool {/// Snapshot the current motion config for `key`, falling back to the/// `"default"` cursor's motion when that key has no own entry yet.pub fn current_motion(key: &str) -> MotionConfig {- RENDER- .lock()- .ok()- .and_then(|g| {- g.as_ref()- .and_then(|m| m.cursor_or_default(key).map(|rs| rs.core.motion.clone()))- })- .unwrap_or_default()+ read_render(|map| map.motion(key)).unwrap_or_default()}-pub fn current_theme_state(- key: &str,-) -> Option<(- String,- String,- String,- Option<String>,- cursor_overlay::CursorVisualState,-)> {- let guard = RENDER.lock().ok()?;- let map = guard.as_ref()?;- let state = map.cursor_or_default(key)?;- let (id, version, profile, fallback) = state.core.active_theme_metadata();- Some((id, version, profile, fallback, state.core.visual.clone()))+pub fn current_theme_state(key: &str) -> Option<cursor_overlay::CursorThemeState> {+ read_render(|map| map.theme_state(key)).flatten()}/// Current screen position, or `None` for an absent or unplaced cursor.pub fn current_position(key: &str) -> Option<(f64, f64)> {- RENDER.lock().ok().and_then(|g| {- g.as_ref()- .and_then(|m| m.cursors.get(key))- .filter(|rs| rs.core.placed)- .map(|rs| rs.core.pos)- })+ read_render(|map| map.placed_position(key)).flatten()}/// Seed a brand-new (sentinel-positioned) cursor at an on-screen start point