Skip to content

armgit / axuse

Public Imported from GitHub
0

Accessibility-first browser & computer use for AI agents — one MCP server for every harness (Codex, Claude Code, OpenCode, Antigravity). Instruments the DOM/accessibility tree via CDP instead of screenshot loops: diffed snapshots, stable element refs, trusted input, typed errors. Zero dependencies

desktop: add pointer event timing controls

tfusion6665f16b1ffa5f7
diff --git a/src/desktop/macos.js b/src/desktop/macos.js
index 54acbe0..d9f6395 100644
--- a/src/desktop/macos.js
+++ b/src/desktop/macos.js
@@ -75,6 +75,13 @@ function finite(value, name) {
return value;
}
+function pointerInterval(value) {
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1000) {
+ throw new AxError(Codes.COMMAND_REJECTED, "interval_ms must be between 0 and 1000", { retry: "never" });
+ }
+ return value;
+}
+
function positive(value, name) {
const n = finite(value, name);
if (n < 1) throw new AxError(Codes.BAD_COORDS, `${name} must be ≥ 1`, { retry: "never" });
@@ -172,7 +179,7 @@ function mouseButtonFor(kind) {
: MOUSE_BUTTONS.left;
}
-function nativeEventCode(events, { post = true } = {}) {
+function nativeEventCode(events, { post = true, intervalMs = 0 } = {}) {
const body = events.map((item, i) => {
if (item.kind === "scroll") {
const horizontal = item.dx !== 0;
@@ -195,7 +202,7 @@ function nativeEventCode(events, { post = true } = {}) {
var e${i} = $.CGEventCreateMouseEvent(null, ${event}, $.CGPointMake(${item.x}, ${item.y}), ${button});
if (!e${i}) throw new Error('CGEventCreateMouseEvent returned null');${clickState}
${post ? `$.CGEventPost($.kCGHIDEventTap, e${i});` : ""}`;
- }).join("\n");
+ }).join(intervalMs ? `\ndelay(${intervalMs / 1000});\n` : "\n");
return `
${post ? "ObjC.import('ApplicationServices'); if (!$.AXIsProcessTrusted()) throw new Error('Accessibility permission required');" : ""}
ObjC.import('CoreGraphics');
@@ -715,7 +722,8 @@ export class MacDriver {
return { action_sent: "sent" };
}
- async click({ x, y, button = "left", double = false } = {}) {
+ async click({ x, y, button = "left", double = false, interval_ms = 0 } = {}) {
+ interval_ms = pointerInterval(interval_ms);
x = finite(x, "x"); y = finite(y, "y");
const b = buttonSpec(button);
const down = b === "right" ? "rdown" : b === "middle" ? "mdown" : "down";
@@ -727,17 +735,19 @@ export class MacDriver {
{ kind: down, x, y, clickState: 1 }, { kind: up, x, y, clickState: 1 },
];
if (double) events.push({ kind: down, x, y, clickState: 2 }, { kind: up, x, y, clickState: 2 });
- await this._serialized(() => this.jxa(mouseSequenceCode(events)));
+ await this._serialized(() => this.jxa(mouseSequenceCode(events, { intervalMs: interval_ms })));
return { action_sent: "sent" };
}
- async drag({ from, to, steps = 12 } = {}) {
+ async drag({ from, to, steps = 12, interval_ms = 0 } = {}) {
+ interval_ms = pointerInterval(interval_ms);
const start = { x: finite(from?.x, "from.x"), y: finite(from?.y, "from.y") };
const end = { x: finite(to?.x, "to.x"), y: finite(to?.y, "to.y") };
const requestedSteps = finite(steps, "steps");
if (requestedSteps < 1) throw new AxError(Codes.COMMAND_REJECTED, "steps must be ≥ 1", { retry: "never" });
const count = Math.round(requestedSteps);
if (count < 1) throw new AxError(Codes.COMMAND_REJECTED, "steps must round to at least 1", { retry: "never" });
+ if (count * interval_ms > 12000) throw new AxError(Codes.COMMAND_REJECTED, "steps × interval_ms must be ≤ 12000", { retry: "never" });
if (count > 200) throw new AxError(Codes.COMMAND_REJECTED, "steps must be ≤ 200", { retry: "never" });
await this._serialized(async () => {
let held = false;
@@ -745,15 +755,17 @@ export class MacDriver {
// Hover + press in one script: the gesture stays atomic up to the hold.
await this.jxa(mouseSequenceCode([
{ kind: "moved", x: start.x, y: start.y }, { kind: "down", x: start.x, y: start.y },
- ]));
+ ], { intervalMs: interval_ms }));
held = true;
+ if (interval_ms) await new Promise((resolve) => setTimeout(resolve, interval_ms));
const moves = [];
for (let i = 1; i <= count; i++) moves.push({
kind: "drag",
x: Math.round(start.x + ((end.x - start.x) * i) / count),
y: Math.round(start.y + ((end.y - start.y) * i) / count),
});
- await this.jxa(mouseSequenceCode(moves));
+ await this.jxa(mouseSequenceCode(moves, { intervalMs: interval_ms }));
+ if (interval_ms) await new Promise((resolve) => setTimeout(resolve, interval_ms));
} finally {
// The release runs in the SAME chain slot — a held button can never
// leak past this call, and no queued gesture can interleave with it.
diff --git a/src/mcp/manifest.js b/src/mcp/manifest.js
index 2987e0a..18541a9 100644
--- a/src/mcp/manifest.js
+++ b/src/mcp/manifest.js
@@ -119,6 +119,7 @@ export function desktopTools(driverAvailable, axCapable = false) {
x: { type: "number" }, y: { type: "number" },
button: { type: "string", enum: ["left", "right", "middle"] },
double: { type: "boolean" },
+ interval_ms: { type: "number", minimum: 0, maximum: 1000, description: "macOS: delay between pointer events in milliseconds (default 0)." },
},
required: ["x", "y"], additionalProperties: false,
},
@@ -170,6 +171,8 @@ export function desktopTools(driverAvailable, axCapable = false) {
properties: {
from: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false },
to: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false },
+ steps: { type: "integer", minimum: 1, maximum: 200, description: "Number of interpolated moves (default 12)." },
+ interval_ms: { type: "number", minimum: 0, maximum: 1000, description: "macOS: delay between pointer events in milliseconds (default 0; steps × interval_ms ≤ 12000)." },
},
required: ["from", "to"],
additionalProperties: false,
diff --git a/src/mcp/server.js b/src/mcp/server.js
index 0285c00..d65e659 100644
--- a/src/mcp/server.js
+++ b/src/mcp/server.js
@@ -215,7 +215,7 @@ export async function createServer({ controller = new Controller(), desktop, sol
return { content: [{ type: "text", text: JSON.stringify({ ...info, note }) }] };
}
case "click": {
- const r = await d.click({ x: args.x, y: args.y, button: args.button, double: args.double });
+ const r = await d.click({ x: args.x, y: args.y, button: args.button, double: args.double, interval_ms: args.interval_ms });
return { content: [{ type: "text", text: JSON.stringify(r) }] };
}
case "type":
@@ -237,7 +237,7 @@ export async function createServer({ controller = new Controller(), desktop, sol
case "ax_set":
return { content: [{ type: "text", text: JSON.stringify(await d.axSet({ ref: args.ref, value: args.value, pid: args.pid, name: args.name, front: args.front })) }] };
case "drag":
- return { content: [{ type: "text", text: JSON.stringify(await d.drag({ from: args.from, to: args.to })) }] };
+ return { content: [{ type: "text", text: JSON.stringify(await d.drag({ from: args.from, to: args.to, steps: args.steps, interval_ms: args.interval_ms })) }] };
default:
throw new AxError("INTERNAL", `unroutable desktop op ${tool._op}`);
}
diff --git a/test/desktop.test.js b/test/desktop.test.js
index 350461a..aebbcbb 100644
--- a/test/desktop.test.js
+++ b/test/desktop.test.js
@@ -514,3 +514,16 @@ test("macOS ax ops fail typed without Accessibility (live, denial path)", { skip
(e) => e.code === "DESKTOP_PERMISSION_REQUIRED" && /Accessibility/.test(e.message) === true,
);
});
+
+test("macOS pointer timing is inserted between events and validated", async () => {
+ const calls = [];
+ const d = new MacDriver({ runFn: async (file, args) => { calls.push({ file, args }); return "ok"; } });
+ await d.click({ x: 4, y: 5, interval_ms: 40 });
+ assert.match(calls[0].args.at(-1), /delay\(0\.04\)/);
+ calls.length = 0;
+ await d.drag({ from: { x: 1, y: 2 }, to: { x: 10, y: 20 }, steps: 3, interval_ms: 1 });
+ assert.match(calls[1].args.at(-1), /delay\(0\.001\)/);
+ assert.match(calls[2].args.at(-1), /kCGEventLeftMouseUp/);
+ await assert.rejects(() => d.click({ x: 1, y: 2, interval_ms: -1 }), (e) => e.code === "COMMAND_REJECTED");
+ await assert.rejects(() => d.drag({ from: { x: 1, y: 2 }, to: { x: 3, y: 4 }, steps: 200, interval_ms: 100 }), (e) => e.code === "COMMAND_REJECTED");
+});