Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): draw the GNOME helper cursor's tip on the click point (#6344)

cua-agents[bot]8437ea257a29
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
index 349521b175..59f057351a 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
@@ -1146,7 +1146,8 @@ mod tests {
#[test]
fn bundled_helper_uses_host_owned_modifier_badge_chips() {
assert!(EXTENSION_SOURCE.contains("GetVersion()"));
- assert!(EXTENSION_SOURCE.contains("return 11;"));
+ // v12 draws the tip, not the canvas centre, on the cursor point.
+ assert!(EXTENSION_SOURCE.contains("return 12;"));
assert!(EXTENSION_SOURCE.contains("PlayCursorPath(trackJson)"));
assert!(EXTENSION_SOURCE.contains("SetCursorLook(lookJson)"));
assert!(EXTENSION_SOURCE.contains("function drawCursorHalo"));
@@ -1175,7 +1176,7 @@ mod tests {
assert!(!EXTENSION_SOURCE.contains("function drawModifiers"));
let metadata: serde_json::Value =
serde_json::from_str(EXTENSION_METADATA).expect("valid bundled helper metadata");
- assert_eq!(metadata["version"], 11);
+ assert_eq!(metadata["version"], 12);
for action in [
"idle", "observe", "click", "drag", "scroll", "text", "key", "navigate", "app",
diff --git a/libs/cua-driver/scripts/tests/test_wayland_helper_cursor_paint.mjs b/libs/cua-driver/scripts/tests/test_wayland_helper_cursor_paint.mjs
new file mode 100644
index 0000000000..8e1e96a55d
--- /dev/null
+++ b/libs/cua-driver/scripts/tests/test_wayland_helper_cursor_paint.mjs
@@ -0,0 +1,243 @@
+// Paints the GNOME Shell helper's cursor into a recording Cairo context and
+// checks what lands on screen: the arrow's tip on the cursor point (CUA-3121),
+// and one arrow and halo colour from rest, through a move, to the click.
+import assert from 'node:assert/strict';
+import { register } from 'node:module';
+
+const extension = new URL('../../wayland-helper/winrects@cua/extension.js', import.meta.url);
+
+const actors = [];
+const timers = new Map();
+let nextTimer = 1;
+let currentContext = null;
+
+class Actor {
+ constructor(props = {}) {
+ Object.assign(this, { x: 0, y: 0, opacity: 255, visible: false }, props);
+ this.handlers = {};
+ actors.push(this);
+ }
+ connect(signal, handler) { this.handlers[signal] = handler; return 1; }
+ set_position(x, y) { this.x = x; this.y = y; }
+ ease(props) { if ('x' in props) this.x = props.x; if ('y' in props) this.y = props.y; }
+ show() { this.visible = true; }
+ hide() { this.visible = false; }
+ queue_repaint() {}
+ remove_all_transitions() {}
+ set_pivot_point() {}
+ set_style() {}
+ add_child() {}
+ destroy() {}
+ get_preferred_width() { return [0, 60]; }
+ set_text(text) { this.text = text; }
+ get_text() { return this.text ?? ''; }
+ get_context() { return currentContext; }
+}
+
+// Cairo with a 2D affine matrix, recording each fill with its source colour
+// and the device-space point its path started at.
+class RecordingContext {
+ constructor() {
+ this.m = [1, 0, 0, 1, 0, 0];
+ this.stack = [];
+ this.path = [];
+ this.source = null;
+ this.fills = [];
+ this.gradients = [];
+ this.ops = 0;
+ }
+ apply(x, y) {
+ const [a, b, c, d, e, f] = this.m;
+ return [a * x + c * y + e, b * x + d * y + f];
+ }
+ multiply([a2, b2, c2, d2, e2, f2]) {
+ const [a, b, c, d, e, f] = this.m;
+ this.m = [a * a2 + c * b2, b * a2 + d * b2, a * c2 + c * d2, b * c2 + d * d2, a * e2 + c * f2 + e, b * e2 + d * f2 + f];
+ }
+ save() { this.stack.push([...this.m]); }
+ restore() { this.m = this.stack.pop(); }
+ translate(x, y) { this.multiply([1, 0, 0, 1, x, y]); }
+ scale(x, y) { this.multiply([x, 0, 0, y, 0, 0]); }
+ rotate(r) { this.multiply([Math.cos(r), Math.sin(r), -Math.sin(r), Math.cos(r), 0, 0]); }
+ moveTo(x, y) { this.path.push(this.apply(x, y)); }
+ lineTo(x, y) { this.path.push(this.apply(x, y)); }
+ curveTo(_x1, _y1, _x2, _y2, x, y) { this.path.push(this.apply(x, y)); }
+ arc(x, y) { this.path.push(this.apply(x, y)); }
+ closePath() {}
+ setSourceRGBA(r, g, b, a) { this.source = { rgb: [r, g, b].map((v) => Math.round(v * 255)), a }; }
+ setSource(gradient) { this.source = { gradient }; this.gradients.push(gradient); }
+ setSourceSurface() { this.source = { surface: true }; }
+ paint() {}
+ record() { this.ops++; this.fills.push({ source: this.source, start: this.path[0] }); }
+ fill() { this.record(); this.path = []; }
+ fillPreserve() { this.record(); }
+ stroke() { this.ops++; this.path = []; }
+ strokePreserve() { this.ops++; }
+ setLineWidth() {}
+ setLineCap() {}
+ setLineJoin() {}
+ $dispose() {}
+}
+
+class RadialGradient {
+ constructor() { this.stops = []; }
+ addColorStopRGBA(offset, r, g, b, a) {
+ this.stops.push({ offset, rgb: [r, g, b].map((v) => Math.round(v * 255)), a });
+ }
+}
+
+globalThis.helperMocks = {
+ Gio: {
+ _promisify() {},
+ DBusExportedObject: { wrapJSObject: () => ({ export() {}, unexport() {} }) },
+ DBus: { session: {} }, BusType: { SESSION: 0 }, BusNameOwnerFlags: { REPLACE: 0 },
+ bus_own_name: () => 1, bus_unown_name() {},
+ },
+ GLib: {
+ PRIORITY_DEFAULT: 0,
+ SOURCE_REMOVE: false,
+ SOURCE_CONTINUE: true,
+ get_monotonic_time: () => globalThis.helperClock * 1_000_000,
+ timeout_add: (_priority, _ms, callback) => { const id = nextTimer++; timers.set(id, callback); return id; },
+ source_remove: (id) => { timers.delete(id); },
+ },
+ Shell: { Screenshot: class {} },
+ St: { DrawingArea: Actor, Label: Actor, BoxLayout: Actor },
+ Clutter: { ActorAlign: { CENTER: 0 }, AnimationMode: { EASE_OUT_CUBIC: 0 } },
+ Cogl: {},
+ cairo: {
+ ImageSurface: class { finish() {} },
+ Context: RecordingContext,
+ RadialGradient,
+ Format: { ARGB32: 0 },
+ LineCap: { ROUND: 0 },
+ LineJoin: { ROUND: 0 },
+ },
+ Main: { layoutManager: { addTopChrome() {} } },
+};
+register(`data:text/javascript,${encodeURIComponent(`
+export async function resolve(specifier, context, next) {
+ if (specifier.startsWith('gi://') || specifier.startsWith('resource:///') || specifier === 'cairo')
+ return { url: 'mock:' + specifier, shortCircuit: true };
+ return next(specifier, context);
+}
+export async function load(url, context, next) {
+ if (url.startsWith('mock:')) {
+ const name = url.slice(5);
+ let source;
+ if (name.endsWith('/extension.js')) source = 'export class Extension {}';
+ else if (name.includes('/config.js')) source = 'export const PACKAGE_VERSION = "46"';
+ else if (name.endsWith('/main.js')) source = 'export const layoutManager = globalThis.helperMocks.Main.layoutManager';
+ else source = 'export default globalThis.helperMocks[' + JSON.stringify(name.replace('gi://', '')) + ']';
+ return { format: 'module', source, shortCircuit: true };
+ }
+ const result = await next(url, context);
+ if (new URL(url).pathname.endsWith('/extension.js')) return { ...result, format: 'module' };
+ return result;
+}
+`)}`);
+
+globalThis.helperClock = 100;
+const { default: Extension } = await import(extension);
+const helper = new Extension();
+helper.enable();
+const cursor = actors.find((actor) => actor.handlers.repaint && actor.width === 112);
+assert.ok(cursor, 'the cursor actor');
+
+// Paint one frame at `now`. The arrow body is the opaque fill in the session
+// colour; its path starts at the tip.
+function paint(now) {
+ globalThis.helperClock = now;
+ currentContext = new RecordingContext();
+ cursor.handlers.repaint(cursor);
+ const fill = helper._fillColor;
+ const body = currentContext.fills.find(
+ ({ source }) => source?.a === 1 && source.rgb?.every((v, i) => v === fill[i])
+ );
+ assert.ok(body, 'the arrow body is painted');
+ const [tx, ty] = body.start;
+ return {
+ tip: [cursor.x + tx, cursor.y + ty],
+ body: body.source.rgb,
+ halo: currentContext.gradients.map((g) => g.stops.map((stop) => stop.rgb.join(','))).flat(),
+ };
+}
+const near = (actual, expected, what) =>
+ assert.ok(
+ Math.abs(actual[0] - expected[0]) < 1e-6 && Math.abs(actual[1] - expected[1]) < 1e-6,
+ `${what}: tip at ${actual} instead of ${expected}`
+ );
+
+const session = '#dd71ec';
+helper.SetCursorColor(session);
+const float = { secs: 4, sway: 1.6406, bob: 1.9688, lift: 1.6406, tilt: 0.04363, hold: 0.5, ease_in: 0.8, ease_out: 0.2 };
+helper.SetCursorLook(JSON.stringify({
+ float,
+ halo: { rgb: session, radius: 14, peak: 0.11, stops: [[0, 1], [0.45, 0.55], [1, 0]], body: [2, 11.9] },
+}));
+
+// Rest: placed and settled, so the tip is exactly on the point.
+helper.PlayCursorPath(JSON.stringify({ samples: [[0, 300, 200, 0, 0, 0]], arrival: null, rest: [0, 0, 0] }));
+const rest = paint(100);
+near(rest.tip, [300, 200], 'at rest');
+
+// Travel: the float moves the whole cursor and the heading turns it about
+// its tip, so the tip sits on the planned point plus the float.
+const samples = [
+ [0, 300, 200, 0, 0, 0],
+ [0.2, 600, 450, 1.2, -1.6, 0.6],
+ [0.4, 900, 600, 0, 0, 0],
+ [0.5, 900, 600, 0, 0, 0],
+];
+helper.PlayCursorPath(JSON.stringify({ samples, arrival: 0.4, rest: [0.5, 0.1, 0] }));
+const [frameTimer] = [...timers.entries()].at(-1);
+globalThis.helperClock = 100.2;
+timers.get(frameTimer)();
+const travel = paint(100.2);
+near(travel.tip, [600 + 1.2, 450 - 1.6], 'while travelling');
+
+// Arrival: settled and upright on the destination.
+globalThis.helperClock = 100.4;
+timers.get(frameTimer)();
+near(paint(100.4).tip, [900, 600], 'on arrival');
+globalThis.helperClock = 100.5;
+timers.get(frameTimer)();
+
+// The click lands where the tip points, also for a click that snaps.
+helper.ClickPulse(900, 600);
+const click = paint(100.55);
+near(click.tip, [900, 600], 'clicking the settled point');
+helper.ClickPulse(1234, 567);
+near(paint(100.6).tip, [1234, 567], 'clicking elsewhere');
+
+// One colour throughout: the arrow and the halo never change from rest,
+// through travel, to arrival and the click.
+for (const [what, frame] of Object.entries({ travel, click })) {
+ assert.deepEqual(frame.body, rest.body, `arrow colour ${what}`);
+ assert.deepEqual(frame.halo, rest.halo, `halo colour ${what}`);
+}
+assert.deepEqual(rest.body, [0xdd, 0x71, 0xec]);
+assert.ok(rest.halo.length > 0 && rest.halo.every((rgb) => rgb === '221,113,236'), 'halo in the session colour');
+
+// A self-played click (no `cursor.marks`) squishes the arrow about its tip
+// but paints no click marks.
+const drawOps = () => {
+ currentContext = new RecordingContext();
+ cursor.handlers.repaint(cursor);
+ return currentContext.ops;
+};
+globalThis.helperClock = 101;
+helper.ClickPulse(1234, 567);
+globalThis.helperClock = 101.15;
+near(paint(101.15).tip, [1234, 567], 'mid-squish');
+const selfPlayed = drawOps();
+helper.SetCursorState('idle', '', '', true);
+const idle = drawOps();
+assert.equal(selfPlayed, idle, 'no click marks without cursor.marks');
+// A click cue cua-driver asks for still draws its marks.
+helper.SetCursorState('click', '', '', true);
+globalThis.helperClock = 101.3;
+assert.ok(drawOps() > idle, 'requested click marks are drawn');
+
+helper.disable();
+console.log('helper cursor paint: ok');
diff --git a/libs/cua-driver/scripts/tests/test_wayland_helper_cursor_paint.py b/libs/cua-driver/scripts/tests/test_wayland_helper_cursor_paint.py
new file mode 100644
index 0000000000..d96066cca0
--- /dev/null
+++ b/libs/cua-driver/scripts/tests/test_wayland_helper_cursor_paint.py
@@ -0,0 +1,14 @@
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+
+def test_wayland_helper_cursor_paint():
+ node = shutil.which("node")
+ if node is None:
+ pytest.fail("Node >= 20.6 is required for the helper cursor paint test")
+ subprocess.run(
+ [node, str(Path(__file__).with_suffix(".mjs"))], check=True
+ )
diff --git a/libs/cua-driver/scripts/tests/test_wayland_helper_safety.mjs b/libs/cua-driver/scripts/tests/test_wayland_helper_safety.mjs
index 89f8ac639d..7b13b1508a 100644
--- a/libs/cua-driver/scripts/tests/test_wayland_helper_safety.mjs
+++ b/libs/cua-driver/scripts/tests/test_wayland_helper_safety.mjs
@@ -69,7 +69,7 @@ for (const version of ['50', '50.1', '50.beta', '50.rc', '51', '45', '46', '47',
assert.deepEqual(counts, {});
} else {
helper.enable();
- assert.equal(helper.GetVersion(), 11);
+ assert.equal(helper.GetVersion(), 12);
assert.equal(counts.export, 1);
assert.equal(counts.own, 1);
assert.equal(counts.actor, 5);
diff --git a/libs/cua-driver/wayland-helper/winrects@cua/extension.js b/libs/cua-driver/wayland-helper/winrects@cua/extension.js
index b8208f4be1..f57f7c24a6 100644
--- a/libs/cua-driver/wayland-helper/winrects@cua/extension.js
+++ b/libs/cua-driver/wayland-helper/winrects@cua/extension.js
@@ -480,11 +480,13 @@ function cursorBodyMotion(progress, action) {
return { dx, dy, rotation, scale };
}
+// About the tip, like the Rust renderer's click squish, so a press never
+// moves the point the arrow clicks.
function applyCursorBodyMotion(cr, motion) {
- cr.translate(64 + motion.dx, 64 + motion.dy);
+ cr.translate(TIP_X + motion.dx, TIP_Y + motion.dy);
cr.rotate(motion.rotation);
cr.scale(motion.scale, motion.scale);
- cr.translate(-64, -64);
+ cr.translate(-TIP_X, -TIP_Y);
}
function drawCursorGlow(cr, progress, action, glowSurface) {
@@ -902,6 +904,8 @@ export default class WinRectsExtension extends Extension {
null
);
this._action = 'idle';
+ // Whether cua-driver asked for the current action's cue.
+ this._cueRequested = false;
this._delivery = '';
this._target = '';
this._actionStarted = nowSeconds();
@@ -929,9 +933,13 @@ export default class WinRectsExtension extends Extension {
const progress = (elapsed % duration) / duration;
const pose = this._pose(nowSeconds());
cr.save();
+ // The actor is placed with its centre on the cursor point (see
+ // _placeCursor), so the tip, not the canvas centre, goes there. Mapping
+ // the canvas centre drew the tip (55, 30) about 3 px left of and 11 px
+ // above the click point.
cr.translate(ACTOR_CENTER, ACTOR_CENTER);
cr.scale(SCALE, SCALE);
- cr.translate(-CANVAS_SIZE / 2, -CANVAS_SIZE / 2);
+ cr.translate(-TIP_X, -TIP_Y);
// The float moves the whole cursor; the rotation (heading plus the
// float tilt) turns it about its tip.
cr.translate(pose.dx / SCALE, pose.dy / SCALE);
@@ -940,7 +948,10 @@ export default class WinRectsExtension extends Extension {
cr.translate(TIP_X, TIP_Y);
cr.rotate(pose.rotation);
cr.translate(-TIP_X, -TIP_Y);
- drawActionCue(cr, this._action, progress, this._fillColor);
+ // A click the helper plays by itself (ClickPulse) only squishes the
+ // arrow; its marks are cua-driver's call (`cursor.marks`).
+ if (this._action !== 'click' || this._cueRequested)
+ drawActionCue(cr, this._action, progress, this._fillColor);
drawCursorBody(cr, progress, this._action, this._fillColor);
cr.restore();
cr.$dispose();
@@ -1080,7 +1091,7 @@ export default class WinRectsExtension extends Extension {
}
}
GetVersion() {
- return 11;
+ return 12;
}
GetRects() {
const actors = global.get_window_actors();
@@ -1274,8 +1285,10 @@ export default class WinRectsExtension extends Extension {
this._cursor.set_position(x - ACTOR_CENTER, y - ACTOR_CENTER);
this._positionBadge(x, y, 0);
}
- if (['idle', 'navigate', 'click'].includes(this._action))
+ if (['idle', 'navigate', 'click'].includes(this._action)) {
this._setCursorState('click', this._delivery, this._target);
+ this._cueRequested = false;
+ }
this._settleFloat();
this._cursor.show();
this._cursor.queue_repaint();
@@ -1378,6 +1391,7 @@ export default class WinRectsExtension extends Extension {
if (!ACTIONS.has(action)) return;
if (active) {
this._setCursorState(action, delivery, target);
+ this._cueRequested = true;
this._settleFloat();
} else if (this._action === action && !ONE_SHOT_ACTIONS.has(action)) {
this._endingAt = nowSeconds() + 0.4;
diff --git a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json
index 99870afafe..477402f8fd 100644
--- a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json
+++ b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json
@@ -3,5 +3,5 @@
"name": "WinRects",
"description": "Expose window geometry, capture, cursor, and verified activation for cua-driver on Wayland.",
"shell-version": ["45", "46", "47", "48", "49"],
- "version": 11
+ "version": 12
}