Accessibility-first browser & computer use for AI agents — one MCP server for every harness (Codex, Claude Code, OpenCode, Antigravity). Instruments the DOM/accessibility tree via CDP instead of screenshot loops: diffed snapshots, stable element refs, trusted input, typed errors. Zero dependencies
JavaScript agentic-automationai-agentsbrowser-automationbrowser-usecdpchrome-devtools-protocolcomputer-usecuamcpmodel-context-protocolrpaweb-automation
desktop: add pointer event timing controls
tfusion6665f16b1ffa5f7
diff --git a/src/desktop/macos.js b/src/desktop/macos.jsindex 54acbe0..d9f6395 100644--- a/src/desktop/macos.js+++ b/src/desktop/macos.js@@ -75,6 +75,13 @@ function finite(value, name) {return value;}+function pointerInterval(value) {+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1000) {+ throw new AxError(Codes.COMMAND_REJECTED, "interval_ms must be between 0 and 1000", { retry: "never" });+ }+ return value;+}+function positive(value, name) {const n = finite(value, name);if (n < 1) throw new AxError(Codes.BAD_COORDS, `${name} must be ≥ 1`, { retry: "never" });@@ -172,7 +179,7 @@ function mouseButtonFor(kind) {: MOUSE_BUTTONS.left;}-function nativeEventCode(events, { post = true } = {}) {+function nativeEventCode(events, { post = true, intervalMs = 0 } = {}) {const body = events.map((item, i) => {if (item.kind === "scroll") {const horizontal = item.dx !== 0;@@ -195,7 +202,7 @@ function nativeEventCode(events, { post = true } = {}) {var e${i} = $.CGEventCreateMouseEvent(null, ${event}, $.CGPointMake(${item.x}, ${item.y}), ${button});if (!e${i}) throw new Error('CGEventCreateMouseEvent returned null');${clickState}${post ? `$.CGEventPost($.kCGHIDEventTap, e${i});` : ""}`;- }).join("\n");+ }).join(intervalMs ? `\ndelay(${intervalMs / 1000});\n` : "\n");return `${post ? "ObjC.import('ApplicationServices'); if (!$.AXIsProcessTrusted()) throw new Error('Accessibility permission required');" : ""}ObjC.import('CoreGraphics');@@ -715,7 +722,8 @@ export class MacDriver {return { action_sent: "sent" };}- async click({ x, y, button = "left", double = false } = {}) {+ async click({ x, y, button = "left", double = false, interval_ms = 0 } = {}) {+ interval_ms = pointerInterval(interval_ms);x = finite(x, "x"); y = finite(y, "y");const b = buttonSpec(button);const down = b === "right" ? "rdown" : b === "middle" ? "mdown" : "down";@@ -727,17 +735,19 @@ export class MacDriver {{ kind: down, x, y, clickState: 1 }, { kind: up, x, y, clickState: 1 },];if (double) events.push({ kind: down, x, y, clickState: 2 }, { kind: up, x, y, clickState: 2 });- await this._serialized(() => this.jxa(mouseSequenceCode(events)));+ await this._serialized(() => this.jxa(mouseSequenceCode(events, { intervalMs: interval_ms })));return { action_sent: "sent" };}- async drag({ from, to, steps = 12 } = {}) {+ async drag({ from, to, steps = 12, interval_ms = 0 } = {}) {+ interval_ms = pointerInterval(interval_ms);const start = { x: finite(from?.x, "from.x"), y: finite(from?.y, "from.y") };const end = { x: finite(to?.x, "to.x"), y: finite(to?.y, "to.y") };const requestedSteps = finite(steps, "steps");if (requestedSteps < 1) throw new AxError(Codes.COMMAND_REJECTED, "steps must be ≥ 1", { retry: "never" });const count = Math.round(requestedSteps);if (count < 1) throw new AxError(Codes.COMMAND_REJECTED, "steps must round to at least 1", { retry: "never" });+ if (count * interval_ms > 12000) throw new AxError(Codes.COMMAND_REJECTED, "steps × interval_ms must be ≤ 12000", { retry: "never" });if (count > 200) throw new AxError(Codes.COMMAND_REJECTED, "steps must be ≤ 200", { retry: "never" });await this._serialized(async () => {let held = false;@@ -745,15 +755,17 @@ export class MacDriver {// Hover + press in one script: the gesture stays atomic up to the hold.await this.jxa(mouseSequenceCode([{ kind: "moved", x: start.x, y: start.y }, { kind: "down", x: start.x, y: start.y },- ]));+ ], { intervalMs: interval_ms }));held = true;+ if (interval_ms) await new Promise((resolve) => setTimeout(resolve, interval_ms));const moves = [];for (let i = 1; i <= count; i++) moves.push({kind: "drag",x: Math.round(start.x + ((end.x - start.x) * i) / count),y: Math.round(start.y + ((end.y - start.y) * i) / count),});- await this.jxa(mouseSequenceCode(moves));+ await this.jxa(mouseSequenceCode(moves, { intervalMs: interval_ms }));+ if (interval_ms) await new Promise((resolve) => setTimeout(resolve, interval_ms));} finally {// The release runs in the SAME chain slot — a held button can never// leak past this call, and no queued gesture can interleave with it.diff --git a/src/mcp/manifest.js b/src/mcp/manifest.jsindex 2987e0a..18541a9 100644--- a/src/mcp/manifest.js+++ b/src/mcp/manifest.js@@ -119,6 +119,7 @@ export function desktopTools(driverAvailable, axCapable = false) {x: { type: "number" }, y: { type: "number" },button: { type: "string", enum: ["left", "right", "middle"] },double: { type: "boolean" },+ interval_ms: { type: "number", minimum: 0, maximum: 1000, description: "macOS: delay between pointer events in milliseconds (default 0)." },},required: ["x", "y"], additionalProperties: false,},@@ -170,6 +171,8 @@ export function desktopTools(driverAvailable, axCapable = false) {properties: {from: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false },to: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false },+ steps: { type: "integer", minimum: 1, maximum: 200, description: "Number of interpolated moves (default 12)." },+ interval_ms: { type: "number", minimum: 0, maximum: 1000, description: "macOS: delay between pointer events in milliseconds (default 0; steps × interval_ms ≤ 12000)." },},required: ["from", "to"],additionalProperties: false,diff --git a/src/mcp/server.js b/src/mcp/server.jsindex 0285c00..d65e659 100644--- a/src/mcp/server.js+++ b/src/mcp/server.js@@ -215,7 +215,7 @@ export async function createServer({ controller = new Controller(), desktop, solreturn { content: [{ type: "text", text: JSON.stringify({ ...info, note }) }] };}case "click": {- const r = await d.click({ x: args.x, y: args.y, button: args.button, double: args.double });+ const r = await d.click({ x: args.x, y: args.y, button: args.button, double: args.double, interval_ms: args.interval_ms });return { content: [{ type: "text", text: JSON.stringify(r) }] };}case "type":@@ -237,7 +237,7 @@ export async function createServer({ controller = new Controller(), desktop, solcase "ax_set":return { content: [{ type: "text", text: JSON.stringify(await d.axSet({ ref: args.ref, value: args.value, pid: args.pid, name: args.name, front: args.front })) }] };case "drag":- return { content: [{ type: "text", text: JSON.stringify(await d.drag({ from: args.from, to: args.to })) }] };+ return { content: [{ type: "text", text: JSON.stringify(await d.drag({ from: args.from, to: args.to, steps: args.steps, interval_ms: args.interval_ms })) }] };default:throw new AxError("INTERNAL", `unroutable desktop op ${tool._op}`);}diff --git a/test/desktop.test.js b/test/desktop.test.jsindex 350461a..aebbcbb 100644--- a/test/desktop.test.js+++ b/test/desktop.test.js@@ -514,3 +514,16 @@ test("macOS ax ops fail typed without Accessibility (live, denial path)", { skip(e) => e.code === "DESKTOP_PERMISSION_REQUIRED" && /Accessibility/.test(e.message) === true,);});++test("macOS pointer timing is inserted between events and validated", async () => {+ const calls = [];+ const d = new MacDriver({ runFn: async (file, args) => { calls.push({ file, args }); return "ok"; } });+ await d.click({ x: 4, y: 5, interval_ms: 40 });+ assert.match(calls[0].args.at(-1), /delay\(0\.04\)/);+ calls.length = 0;+ await d.drag({ from: { x: 1, y: 2 }, to: { x: 10, y: 20 }, steps: 3, interval_ms: 1 });+ assert.match(calls[1].args.at(-1), /delay\(0\.001\)/);+ assert.match(calls[2].args.at(-1), /kCGEventLeftMouseUp/);+ await assert.rejects(() => d.click({ x: 1, y: 2, interval_ms: -1 }), (e) => e.code === "COMMAND_REJECTED");+ await assert.rejects(() => d.drag({ from: { x: 1, y: 2 }, to: { x: 3, y: 4 }, steps: 200, interval_ms: 100 }), (e) => e.code === "COMMAND_REJECTED");+});