Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Agent cursor setting tolerates a driver without an overlay; live click speed check (LPT-916)
alexshpunt-clanker[bot]7e9ebf4a2e70
diff --git a/docs/complete-input.md b/docs/complete-input.mdindex 1116089..e10bcd8 100644--- a/docs/complete-input.md+++ b/docs/complete-input.md@@ -4,7 +4,7 @@ See [the resource guide](computer-resources.md) for the current addresses, viewsSend input as one Insert action into a current window overview. CUA remains the only input backend. A display capture or detail cannot ground input.-CUA draws an agent cursor over the desktop and, by default, waits for it to glide to each new point before a pointer action: about 600 ms per 300 px. When the driver starts, the adapter sets that glide to 1 ms, so the agent cursor jumps to each point and still shows the click. A background click on a WinForms button then takes about 22 ms instead of 640 ms. The driver's own pauses between press and release stay: they are what makes double and right clicks take about 200 and 90 ms. If the driver refuses the setting or does not apply it, startup fails before any input.+CUA draws an agent cursor over the desktop and, by default, waits for it to glide to each new point before a pointer action: about 600 ms per 300 px. When the driver starts, the adapter sets that glide to 1 ms, so the agent cursor jumps to each point and still shows the click. A background click on a WinForms button then takes about 22 ms instead of 640 ms. The driver's own pauses between press and release stay: they are what makes double and right clicks take about 200 and 90 ms. If the driver refuses the setting or does not apply it, startup fails before any input. An isolated session's driver runs without the overlay: it draws no cursor and so never waits for one.## Single actionsdiff --git a/src/driver.ts b/src/driver.tsindex c4adc6f..b6a6c88 100644--- a/src/driver.ts+++ b/src/driver.ts@@ -232,18 +232,23 @@ export class McpDriver implements Driver {* Before each pointer action at a new point the driver waits for its agent cursor to glide there: about 600 ms* per 300 px at its default speed. A 1 ms glide makes the cursor jump; 0 would select that speed-based glide.* The setter only queues a render command, so its state is read back, yielding 5 ms between reads.+ * Before each action the driver waits only for a cursor it draws.*/private async jumpCursor(client: Client, signal?: AbortSignal): Promise<void> {const call = async (name: string, args: Record<string, unknown>) =>CallToolResultSchema.parse(await client.callTool({ name, arguments: args }, undefined, { signal, timeout: 15_000 }));if ((await call("set_agent_cursor_motion", { glide_duration_ms: CURSOR_GLIDE_MS })).isError)throw new Error("The driver refused the agent cursor glide setting.");+ let state: Record<string, unknown> | undefined;for (let read = 0; read < 6; read++) {- const state = await call("get_agent_cursor_state", {});- if ((state.structuredContent?.motion as { glide_duration_ms?: unknown } | undefined)?.glide_duration_ms === CURSOR_GLIDE_MS) return;+ state = (await call("get_agent_cursor_state", {})).structuredContent;+ if ((state?.motion as { glide_duration_ms?: unknown } | undefined)?.glide_duration_ms === CURSOR_GLIDE_MS) return;await delay(5, undefined, { signal });}- throw new Error("The driver did not apply the agent cursor glide setting.");+ // A driver without an overlay (an isolated session runs with --no-overlay) drops the setting and reports the+ // cursor disabled; it draws no cursor and so never waits for one.+ if (state?.enabled === false) return;+ throw new Error(`The driver did not apply the agent cursor glide setting: ${JSON.stringify({ enabled: state?.enabled, motion: state?.motion })}`);}/**diff --git a/test/agent-cursor.test.ts b/test/agent-cursor.test.tsindex 9bd0b79..470eaf3 100644--- a/test/agent-cursor.test.ts+++ b/test/agent-cursor.test.ts@@ -7,7 +7,7 @@ import { McpDriver } from "../src/driver.ts";const dirs: string[] = [];afterEach(async () => { await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });-async function start(mode: "applies" | "refuses" | "ignores") {+async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay") {const dir = await mkdtemp(path.join(tmpdir(), "agent-cursor-"));dirs.push(dir);const log = path.join(dir, "calls.jsonl");@@ -34,6 +34,14 @@ test("a refused cursor setting stops the driver before any input", async () => {expect((await calls()).map(call => call.name)).toEqual(["set_agent_cursor_motion"]);});+test("a driver without an overlay draws no cursor, so an unapplied setting does not stop it", async () => {+ const { driver, calls } = await start("no-overlay");+ try {+ expect((await driver.call("list_windows", {})).isError).toBeFalsy();+ expect((await calls()).map(call => call.name)).toEqual(["set_agent_cursor_motion", ...Array(6).fill("get_agent_cursor_state"), "list_windows"]);+ } finally { await driver.close(); }+});+test("a cursor setting that never applies stops the driver after six reads", async () => {const { driver, calls } = await start("ignores");await expect(driver.call("list_windows", {})).rejects.toThrow("did not apply the agent cursor glide setting");diff --git a/test/fixtures/agent-cursor-driver.mjs b/test/fixtures/agent-cursor-driver.mjsindex af1ed37..4f33f1b 100644--- a/test/fixtures/agent-cursor-driver.mjs+++ b/test/fixtures/agent-cursor-driver.mjs@@ -3,7 +3,8 @@ import { Server } from "@modelcontextprotocol/sdk/server/index.js";import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";-// Deterministic MCP peer for the startup cursor setting. Mode: applies, refuses, or ignores the glide.+// Deterministic MCP peer for the startup cursor setting. Mode: applies, refuses or ignores the glide, or has no+// overlay: then the glide stays unset and the cursor is reported disabled.const [log, mode] = process.argv.slice(2);let glide = 0;const server = new Server({ name: "cua-driver", version: "0.33.2" }, { capabilities: { tools: {} } });@@ -14,7 +15,8 @@ server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, argumenif (mode === "refuses") return { content: [{ type: "text", text: "refused" }], isError: true };if (mode === "applies") glide = args.glide_duration_ms;}- if (name === "get_agent_cursor_state") return { content: [], structuredContent: { motion: { glide_duration_ms: glide } } };+ if (name === "get_agent_cursor_state")+ return { content: [], structuredContent: { enabled: mode !== "no-overlay", motion: { glide_duration_ms: glide } } };return { content: [], structuredContent: { windows: [] } };});await server.connect(new StdioServerTransport());diff --git a/test/live/click-speed.live.ts b/test/live/click-speed.live.tsnew file mode 100644index 0000000..952029c--- /dev/null+++ b/test/live/click-speed.live.ts@@ -0,0 +1,41 @@+import { afterAll, expect, test } from "vitest";+import { addresses, connectLive, nativeForm, openWindow, text, WINDOWS_POWERSHELL_PATH as POWERSHELL } from "./remote.ts";++// LPT-916: the driver used to wait for its agent cursor to glide to each new point, about 600 ms per click.+// The cursor now jumps, so a run of clicks between two controls takes as long as the clicks themselves.+const live = await connectLive();+let pid = 0;+afterAll(async () => { if (pid) await live.kill(pid); await live.close(); });++type Element = { role?: string | null; name?: string | null; bounds?: { window_image?: { x: number; y: number; width: number; height: number } } };+const centre = (element: Element | undefined) => {+ const box = element?.bounds?.window_image;+ if (!box) throw new Error(`No overview bounds for ${JSON.stringify(element)}`);+ return { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };+};++test("ten background clicks alternating between two controls each finish in under 100 ms", async () => {+ const title = `Click speed ${Date.now()}`;+ const window = await openWindow(live, live.machine, POWERSHELL, nativeForm(title), title);+ pid = (JSON.parse(JSON.stringify((await live.search(live.windows, title)).details.state)).windows as { pid: number }[])[0].pid;+ expect(addresses(await live.search(live.windows, title), "window")).toContain(window);+ const view = await live.read(window, ["full"]);+ const elements = view.details.ui!.elements as Element[];+ const button = centre(elements.find(e => e.role === "Button" && /^Count/.test(e.name ?? "")));+ const field = centre(elements.find(e => e.role === "Edit"));+ const sequence = Array.from({ length: 10 }, (_, i) => ({ action: "click", ...(i % 2 ? field : button) }));+ const done = await live.insert(`computer:observation/${view.details.observation_id}`, { action: "sequence", sequence, successor: false });+ expect(done.details.status, text(done)).toBe("delivered");+ type Step = { dispatch_ms: number; completion_ms: number; action_capability: { route: string } };+ const steps = (done.details.state as { sequence: { steps: Step[] } }).sequence.steps;+ const ms = (step: Step) => step.completion_ms - step.dispatch_ms;+ console.log(`click times, ms: ${steps.map(step => `${Math.round(ms(step))} (${step.action_capability.route})`).join(", ")}`);+ // Times include the SSH round trip. The button is invoked through UIA; the text box gets posted mouse messages,+ // which carry the driver's own 35 ms press and 50 ms settle pauses. A glide alone used to add over 300 ms.+ const invoked = steps.filter(step => step.action_capability.route === "accessibility").map(ms).sort((a, b) => a - b);+ expect(invoked).toHaveLength(5);+ expect(invoked[2]).toBeLessThan(100);+ expect(Math.max(...steps.map(ms))).toBeLessThan(300);+ const after = await live.read(window, ["ui"]);+ expect((after.details.ui!.elements as Element[]).some(e => e.role === "Button" && e.name === "Count 5"), text(after)).toBe(true);+});