Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): anchor theme transforms at hotspot (#4246)
lorenzozaneb0ebd915daf8
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rsindex 37d1655041..36f37172d2 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs@@ -383,18 +383,42 @@ pub enum OverlayCommand {ShowFocusRect(Option<[f64; 4]>),}+/// Distance, in points, between a cursor's pointer point and its anchor.+///+/// `RenderStateCore::pos` is the anchor that path motion, the session badge,+/// and platform damage regions follow. The theme hotspot is drawn at the+/// pointer point, `POINTER_ANCHOR_OFFSET` points from the anchor opposite the+/// heading, so a cursor anchored by [`anchor_for_pointer`] draws its tip on+/// the requested coordinate at every heading and backing scale.+pub const POINTER_ANCHOR_OFFSET: f64 = 16.0;++/// Anchor that places a cursor's hotspot on `(x, y)` at `heading`.+pub fn anchor_for_pointer(x: f64, y: f64, heading: f64) -> (f64, f64) {+ (+ x + heading.cos() * POINTER_ANCHOR_OFFSET,+ y + heading.sin() * POINTER_ANCHOR_OFFSET,+ )+}++/// Pointer point, where the theme hotspot is drawn, for an anchor at `heading`.+pub fn pointer_for_anchor(x: f64, y: f64, heading: f64) -> (f64, f64) {+ (+ x - heading.cos() * POINTER_ANCHOR_OFFSET,+ y - heading.sin() * POINTER_ANCHOR_OFFSET,+ )+}+/// Build the shared overlay command for one native pointer position.///-/// Native drag implementations report the actual event coordinate while the-/// cursor artwork is centred 16 points down-right so its tip lands on that-/// coordinate. Keeping this transform here prevents platform-specific drag-/// loops from drifting apart.+/// Native drag implementations report the actual event coordinate. Anchoring+/// it here keeps the theme hotspot on that coordinate and prevents+/// platform-specific drag loops from drifting apart.pub fn track_pointer_command(x: f64, y: f64) -> OverlayCommand {- const CLICK_OFFSET: f64 = 16.0;let heading = std::f64::consts::FRAC_PI_4;+ let (x, y) = anchor_for_pointer(x, y, heading);OverlayCommand::SnapTo {- x: x + heading.cos() * CLICK_OFFSET,- y: y + heading.sin() * CLICK_OFFSET,+ x,+ y,heading_radians: Some(heading),}}@@ -446,22 +470,6 @@ mod pointer_tracking_tests {assert!(!sibling.get());}- #[test]- fn tracked_artwork_keeps_its_tip_on_the_native_pointer() {- let OverlayCommand::SnapTo {- x,- y,- heading_radians: Some(heading),- } = track_pointer_command(120.0, 80.0)- else {- panic!("pointer tracking must produce an anchored snap");- };- // The artwork centre sits 16 points down-right of the tip at 45 degrees.- assert_eq!(heading, std::f64::consts::FRAC_PI_4);- assert!((x - 131.313_708_498_984_76).abs() < 1e-9, "x = {x}");- assert!((y - 91.313_708_498_984_76).abs() < 1e-9, "y = {y}");- }-#[test]fn session_cleanup_removes_named_cursor_but_preserves_anonymous_default() {let registry = CursorRegistry::new();diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rsindex 0774a11bc6..264c427615 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs@@ -644,14 +644,11 @@ impl RenderStateCore {end_heading_radians,} => {let reveal_badge = !self.is_revealed();- // Apply click offset (16 pt along end_heading) before planning,- // matching Swift `moveTo(point:endAngleRadians:)`:- // tx = clickPoint.x + cos(endAngle) * clickOffset- // ty = clickPoint.y + sin(endAngle) * clickOffset- const CLICK_OFFSET: f64 = 16.0;+ // Plan the anchor, not the pointer point, so the hotspot lands+ // on `(x, y)` once the cursor settles at `end_heading`+ // (Swift `moveTo(point:endAngleRadians:)` click offset).let turn_radius = self.motion.turn_radius;- let tx = x + end_heading_radians.cos() * CLICK_OFFSET;- let ty = y + end_heading_radians.sin() * CLICK_OFFSET;+ let (tx, ty) = crate::anchor_for_pointer(x, y, end_heading_radians);// macOS-only: if the cursor is still at the initial off-screen// sentinel, snap it to the offset target so the path starts on-screen.@@ -713,20 +710,12 @@ impl RenderStateCore {}OverlayCommand::ClickPulse { x, y } => {let reveal_badge = !self.is_revealed();- if click_pulse_sentinel_only {- // macOS: only snap position on first placement (sentinel state).- // After that the cursor stays where the animation landed.- if self.pos.0 < -50.0 {- // Apply same click offset so tip lands at click point.- const CLICK_OFFSET: f64 = 16.0;- let angle = std::f64::consts::FRAC_PI_4;- self.pos = (- x + angle.cos() * CLICK_OFFSET,- y + angle.sin() * CLICK_OFFSET,- );- }- } else {- self.pos = (x, y);+ // macOS only snaps on first placement (sentinel state); after+ // that the cursor stays where the animation landed. Windows+ // and Linux always snap. Both anchor the click point so the+ // hotspot stays on it instead of jumping by the anchor offset.+ if !click_pulse_sentinel_only || self.pos.0 < -50.0 {+ self.pos = crate::anchor_for_pointer(x, y, self.heading);}self.click_t = Some(0.0);if matches!(@@ -913,6 +902,9 @@ pub fn paint_cursor(// first, then scale into pixmap pixels.let (px, py) = ((core.pos.0 - origin_x) * s, (core.pos.1 - origin_y) * s);let heading = core.heading;+ // Themes pivot on their hotspot, which is drawn at the pointer point.+ let (tip_x, tip_y) = crate::pointer_for_anchor(core.pos.0, core.pos.1, heading);+ let (tip_x, tip_y) = ((tip_x - origin_x) * s, (tip_y - origin_y) * s);let alpha_scale = core.idle_alpha as f32;// --- Focus rect highlight (macOS only — others pass None) ---@@ -974,8 +966,8 @@ pub fn paint_cursor(pm,theme,&core.visual,- px as f32,- py as f32,+ tip_x as f32,+ tip_y as f32,heading as f32,backing_scale.max(1.0),alpha_scale,@@ -988,8 +980,8 @@ pub fn paint_cursor(crate::theme::paint_default_theme_with_fill(pm,&core.visual,- px as f32,- py as f32,+ tip_x as f32,+ tip_y as f32,heading as f32,backing_scale.max(1.0),alpha_scale,@@ -1095,7 +1087,10 @@ mod session_badge_and_action_tests {core.tick_motion(2.0);assert!(core.is_revealed());- assert_eq!(core.pos, (40.0, 60.0));+ assert_eq!(+ core.pos,+ crate::anchor_for_pointer(40.0, 60.0, core.heading)+ );assert_eq!(core.idle_alpha, 1.0);}@@ -1438,3 +1433,153 @@ mod backing_scale_tests {}}}++#[cfg(test)]+mod pointer_anchor_tests {+ use super::*;+ use crate::{+ track_pointer_command, CompiledAnimation, CompiledDrawCommand, CompiledFrame,+ CompiledGeometry, CompiledTheme, CompiledTransform, CursorAction, CursorConfig,+ };+ use std::f64::consts::{FRAC_PI_2, FRAC_PI_4};++ /// A theme whose only artwork is a small disc centred on a non-central+ /// hotspot, so the disc centroid is the painted hotspot.+ fn hotspot_marker_theme() -> Arc<CompiledTheme> {+ let animation = CompiledAnimation {+ still_frame: 0,+ frames: vec![CompiledFrame {+ commands: vec![CompiledDrawCommand {+ geometries: vec![CompiledGeometry::Ellipse {+ center: [55.0, 30.0],+ size: [8.0, 8.0],+ }],+ transform: CompiledTransform::default(),+ opacity: 1.0,+ fill: Some([255, 0, 0, 255]),+ stroke: None,+ }],+ }],+ };+ Arc::new(CompiledTheme {+ id: "com.example.hotspot".into(),+ name: "Hotspot".into(),+ version: "1.0.0".into(),+ author: "Example Author".into(),+ license: "MIT".into(),+ profile: crate::THEME_PROFILE.into(),+ source_hash: [0; 32],+ hotspot: [55, 30],+ actions: CursorAction::ALL+ .into_iter()+ .map(|action| (action.as_str().to_owned(), animation.clone()))+ .collect(),+ })+ }++ fn cursor() -> RenderStateCore {+ let mut core = RenderStateCore::new(CursorConfig::default());+ core.theme = Some(hotspot_marker_theme());+ core.motion.idle_hide_ms = 0.0;+ core+ }++ /// Painted hotspot in logical points.+ fn painted_hotspot(core: &RenderStateCore, backing_scale: f32) -> (f64, f64) {+ let (width, height) = (+ (400.0 * backing_scale) as u32,+ (300.0 * backing_scale) as u32,+ );+ let pm = render_frame(core, width, height, 0.0, 0.0, None, backing_scale);+ let (weight, x_sum, y_sum) = pm.data().chunks_exact(4).enumerate().fold(+ (0.0, 0.0, 0.0),+ |(weight, x_sum, y_sum), (index, pixel)| {+ let alpha = f64::from(pixel[3]);+ let x = (index % pm.width() as usize) as f64 + 0.5;+ let y = (index / pm.width() as usize) as f64 + 0.5;+ (weight + alpha, x_sum + x * alpha, y_sum + y * alpha)+ },+ );+ assert!(weight > 0.0, "hotspot marker was not painted");+ let scale = f64::from(backing_scale);+ (x_sum / weight / scale, y_sum / weight / scale)+ }++ fn assert_hotspot_at(core: &RenderStateCore, backing_scale: f32, x: f64, y: f64, when: &str) {+ let (hx, hy) = painted_hotspot(core, backing_scale);+ assert!(+ (hx - x).abs() <= 0.5 && (hy - y).abs() <= 0.5,+ "{when}: hotspot painted at ({hx:.2}, {hy:.2}), expected ({x}, {y}) \+ at heading {:.3} and {backing_scale}x",+ core.heading+ );+ }++ fn settle(core: &mut RenderStateCore, macos: bool) {+ for _ in 0..1200 {+ if macos {+ core.tick_swift_constants(1.0 / 60.0);+ } else {+ core.tick_motion(1.0 / 60.0);+ }+ if core.path.is_none() && core.spring.is_none() {+ return;+ }+ }+ panic!("cursor did not settle");+ }++ /// Every pointer-producing command anchors the cursor so the theme hotspot+ /// is painted on the requested coordinate, on both the macOS+ /// (sentinel-only click snap, Swift constants) and Windows/Linux paths.+ #[test]+ fn hotspot_lands_on_requested_pointer_for_move_click_and_drag() {+ for macos in [false, true] {+ for backing_scale in [1.0_f32, 2.0] {+ // First placement from the off-screen sentinel.+ let mut core = cursor();+ core.apply_command_base(+ OverlayCommand::ClickPulse { x: 90.0, y: 70.0 },+ macos,+ macos,+ );+ assert_hotspot_at(&core, backing_scale, 90.0, 70.0, "sentinel click");++ for heading in [FRAC_PI_4, 0.0, FRAC_PI_2, 3.0 * FRAC_PI_4] {+ core.apply_command_base(+ OverlayCommand::MoveTo {+ x: 220.0,+ y: 160.0,+ end_heading_radians: heading,+ },+ macos,+ macos,+ );+ settle(&mut core, macos);+ assert_hotspot_at(&core, backing_scale, 220.0, 160.0, "settled move");++ core.apply_command_base(+ OverlayCommand::ClickPulse { x: 220.0, y: 160.0 },+ macos,+ macos,+ );+ assert_hotspot_at(&core, backing_scale, 220.0, 160.0, "click pulse");++ core.apply_command_base(+ OverlayCommand::MoveTo {+ x: 90.0,+ y: 70.0,+ end_heading_radians: heading,+ },+ macos,+ macos,+ );+ settle(&mut core, macos);+ }++ core.apply_command_base(track_pointer_command(150.0, 110.0), macos, macos);+ assert_hotspot_at(&core, backing_scale, 150.0, 110.0, "tracked drag");+ }+ }+ }+}diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/theme.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/theme.rsindex e9d606916c..624e2aec7b 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/theme.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/theme.rs@@ -196,9 +196,8 @@ pub(crate) fn shared_float_motion(visual: &CursorVisualState) -> (f32, f32, f32)/// Paint one frame of the embedded actions-v2 theme.///-/// `anchor_x/y` is the existing overlay's cursor centre. The Lottie canvas is-/// centred there so the established click offset and path physics remain-/// unchanged. `heading` rotates the artwork around the canvas centre.+/// `anchor_x/y` is where the theme hotspot is drawn. `heading` and the+/// display scale pivot around that hotspot.pub fn paint_default_theme(pm: &mut tiny_skia::Pixmap,visual: &CursorVisualState,diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/theme_artifact.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/theme_artifact.rsindex fbd7f57c8d..473940efee 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/theme_artifact.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/theme_artifact.rs@@ -763,6 +763,9 @@ pub fn paint_compiled_theme(/// Paint a compiled theme, optionally replacing the embedded default's blue/// palette key with a stable per-session fill.///+/// The theme hotspot is drawn at `anchor_x/y`; heading and display scale+/// pivot around it so the tip stays put while the artwork turns.+////// Custom themes pass `None` and retain their authored colors. Only the/// checked-in `cua.default` path passes a tint.#[allow(clippy::too_many_arguments)]@@ -783,10 +786,11 @@ pub fn paint_compiled_theme_with_tint(} else {(0.0, 0.0, 0.0)};- let transform = Transform::from_translate(-64.0, -64.0)- .post_scale(scale, scale)- .post_rotate((heading - std::f32::consts::FRAC_PI_4 + float_rotation).to_degrees())- .post_translate(anchor_x + float_dx * scale, anchor_y + float_dy * scale);+ let transform =+ Transform::from_translate(-f32::from(theme.hotspot[0]), -f32::from(theme.hotspot[1]))+ .post_scale(scale, scale)+ .post_rotate((heading - std::f32::consts::FRAC_PI_4 + float_rotation).to_degrees())+ .post_translate(anchor_x + float_dx * scale, anchor_y + float_dy * scale);let reduced = visual.reduced_motion == crate::ReducedMotion::On;if let Some(animation) = theme.animation_for_action(visual.resolved_action) {draw_layer(@@ -923,4 +927,112 @@ mod tests {let expected: [u8; 32] = Sha256::digest(source).into();assert_eq!(embedded_default_theme().source_hash, expected);}++ fn hotspot_theme(transforms: Vec<CompiledTransform>) -> CompiledTheme {+ let frames = transforms+ .into_iter()+ .map(|transform| CompiledFrame {+ commands: vec![CompiledDrawCommand {+ geometries: vec![CompiledGeometry::Ellipse {+ center: [55.0, 30.0],+ size: [8.0, 8.0],+ }],+ transform,+ opacity: 1.0,+ fill: Some([255, 0, 0, 255]),+ stroke: None,+ }],+ })+ .collect();+ let mut theme = minimal_theme();+ theme.actions.insert(+ "idle".into(),+ CompiledAnimation {+ still_frame: 0,+ frames,+ },+ );+ theme+ }++ fn marker_centroid(pixmap: &tiny_skia::Pixmap) -> (f32, f32) {+ let (weight, x_sum, y_sum) = pixmap.data().chunks_exact(4).enumerate().fold(+ (0.0, 0.0, 0.0),+ |(weight, x_sum, y_sum), (index, pixel)| {+ let alpha = f32::from(pixel[3]);+ (+ weight + alpha,+ x_sum + (index % pixmap.width() as usize) as f32 * alpha,+ y_sum + (index / pixmap.width() as usize) as f32 * alpha,+ )+ },+ );+ assert!(weight > 0.0, "synthetic hotspot marker was not painted");+ (x_sum / weight, y_sum / weight)+ }++ fn render_marker(+ theme: &CompiledTheme,+ heading: f32,+ backing_scale: f32,+ elapsed_secs: f64,+ ) -> (f32, f32) {+ let mut pixmap = tiny_skia::Pixmap::new(256, 256).unwrap();+ let visual = CursorVisualState {+ elapsed_secs,+ ..CursorVisualState::default()+ };+ paint_compiled_theme_with_tint(+ &mut pixmap,+ theme,+ &visual,+ 128.0,+ 128.0,+ heading,+ backing_scale,+ 1.0,+ None,+ );+ marker_centroid(&pixmap)+ }++ #[test]+ fn hotspot_stays_at_cursor_through_heading_scale_and_semantic_frames() {+ let theme = hotspot_theme(vec![CompiledTransform::default(); 4]);+ let headings = [+ std::f32::consts::FRAC_PI_4,+ 0.0,+ std::f32::consts::FRAC_PI_2,+ ];++ for heading in headings {+ for backing_scale in [1.0, 2.0] {+ for frame in 0..4 {+ let elapsed_secs = f64::from(frame) / f64::from(FPS);+ let (x, y) = render_marker(&theme, heading, backing_scale, elapsed_secs);+ assert!(+ (x - 128.0).abs() <= 0.75 && (y - 128.0).abs() <= 0.75,+ "hotspot drifted to ({x:.2}, {y:.2}) at heading={heading}, scale={backing_scale}, frame={frame}"+ );+ }+ }+ }+ }++ #[test]+ fn authored_layer_translation_remains_relative_to_hotspot() {+ let plain = hotspot_theme(vec![CompiledTransform::default()]);+ let translated = hotspot_theme(vec![CompiledTransform {+ position: [6.0, 2.0],+ ..CompiledTransform::default()+ }]);+ let heading = std::f32::consts::FRAC_PI_4;+ let backing_scale = 2.0;+ let origin = render_marker(&plain, heading, backing_scale, 0.0);+ let moved = render_marker(&translated, heading, backing_scale, 0.0);+ let scale = crate::theme::DISPLAY_SIZE * backing_scale / CANVAS as f32;++ assert!((moved.0 - origin.0 - 6.0 * scale).abs() <= 0.35);+ assert!((moved.1 - origin.1 - 2.0 * scale).abs() <= 0.35);+ }}diff --git a/libs/cua-driver/rust/crates/cursor-theme-cli/src/main.rs b/libs/cua-driver/rust/crates/cursor-theme-cli/src/main.rsindex d0aae9ad73..e8ab802099 100644--- a/libs/cua-driver/rust/crates/cursor-theme-cli/src/main.rs+++ b/libs/cua-driver/rust/crates/cursor-theme-cli/src/main.rs@@ -884,6 +884,35 @@ fn compile_animation(bytes: &[u8], id: &str) -> Result<CompiledAnimation> {})}+const PREVIEW_BACKING_SCALE: f32 = CANVAS as f32 / 48.0;++/// Preview pixel where the hotspot must be drawn so the authored canvas stays+/// centred. The renderer pivots on the hotspot rather than the canvas centre.+fn preview_hotspot(theme: &CompiledTheme) -> (f32, f32) {+ let units_to_pixels = cursor_overlay::theme::DISPLAY_SIZE * PREVIEW_BACKING_SCALE+ / cursor_overlay::theme::CANVAS_SIZE;+ let centre = CANVAS as f32 * 0.5;+ let place = |hotspot: u16| centre + (f32::from(hotspot) - centre) * units_to_pixels;+ (place(theme.hotspot[0]), place(theme.hotspot[1]))+}++fn preview_pixmap(theme: &CompiledTheme, visual: &CursorVisualState) -> Result<tiny_skia::Pixmap> {+ let mut pixmap =+ tiny_skia::Pixmap::new(CANVAS, CANVAS).ok_or_else(|| anyhow!("create preview pixmap"))?;+ let (anchor_x, anchor_y) = preview_hotspot(theme);+ cursor_overlay::paint_compiled_theme(+ &mut pixmap,+ theme,+ visual,+ anchor_x,+ anchor_y,+ std::f32::consts::FRAC_PI_4,+ PREVIEW_BACKING_SCALE,+ 1.0,+ );+ Ok(pixmap)+}+fn preview(theme: &CompiledTheme, output: &Path) -> Result<()> {fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?;for name in theme.actions.keys() {@@ -900,18 +929,7 @@ fn preview(theme: &CompiledTheme, output: &Path) -> Result<()> {} else {bail!("unknown compiled action `{name}`");}- let mut pixmap = tiny_skia::Pixmap::new(CANVAS, CANVAS)- .ok_or_else(|| anyhow!("create preview pixmap"))?;- cursor_overlay::paint_compiled_theme(- &mut pixmap,- theme,- &visual,- CANVAS as f32 * 0.5,- CANVAS as f32 * 0.5,- std::f32::consts::FRAC_PI_4,- CANVAS as f32 / 48.0,- 1.0,- );+ let pixmap = preview_pixmap(theme, &visual)?;// tiny-skia stores premultiplied pixels; PNG expects straight RGBA.let pixels = unpremultiply_rgba(pixmap.data().to_vec());let path = output.join(format!("{name}.png"));@@ -950,6 +968,25 @@ mod tests {use std::io::Write;use zip::{write::SimpleFileOptions, ZipWriter};+ #[test]+ fn preview_keeps_the_default_artwork_inside_the_canvas() {+ let theme = cursor_overlay::embedded_default_theme();+ let visual = CursorVisualState {+ reduced_motion: ReducedMotion::On,+ ..CursorVisualState::default()+ };+ let pixmap = preview_pixmap(&theme, &visual).unwrap();+ let edge = CANVAS - 1;+ let clipped = pixmap+ .data()+ .chunks_exact(4)+ .enumerate()+ .filter(|(_, pixel)| pixel[3] > 0)+ .map(|(index, _)| (index as u32 % CANVAS, index as u32 / CANVAS))+ .find(|&(x, y)| x == 0 || y == 0 || x == edge || y == edge);+ assert_eq!(clipped, None, "preview artwork reaches the canvas edge");+ }+#[test]fn preview_unpremultiplication_is_bounded() {let straight = unpremultiply_rgba(vec![100, 50, 25, 128, 0, 0, 0, 0]);diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rsindex f51e2a2dc0..1c5ac2ff4a 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs@@ -3962,7 +3962,10 @@ mod tests {assert!(arrived.is_empty());assert!(had_msg);let cursor = &map.cursors["default"].core;- assert_eq!(cursor.pos, (40.0, 50.0));+ assert_eq!(+ cursor.pos,+ cursor_overlay::anchor_for_pointer(40.0, 50.0, cursor.heading)+ );assert_eq!(cursor.click_t, Some(0.0));}