Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
fix: preserve native visuals across startup retries
alexshpunt-clanker[bot]cfc6bb497a25
diff --git a/src/driver.ts b/src/driver.tsindex 08bd346..9a81903 100644--- a/src/driver.ts+++ b/src/driver.ts@@ -245,9 +245,10 @@ export class McpDriver implements Driver {this.nativeVisualsAvailable = false;}} else await this.jumpCursor(client, signal);+ signal?.throwIfAborted();return client;} catch (error) {- await this.close();+ await this.disconnect();throw new Error(`Driver startup failed: ${error instanceof Error ? error.message : error}\n${this.stderr}`);}}@@ -379,6 +380,11 @@ export class McpDriver implements Driver {/** EOF owns shutdown; the SDK escalates only for this child if it does not exit. */async close(): Promise<void> {this.presentation.close();+ await this.disconnect();+ }++ /** Startup cleanup is retryable; only final close disposes presentation and its subscriptions. */+ private async disconnect(): Promise<void> {this.nativeVisualsAvailable = false;const client = this.client;const transport = this.transport;diff --git a/test/agent-cursor.test.ts b/test/agent-cursor.test.tsindex d781c8e..b1bbc87 100644--- a/test/agent-cursor.test.ts+++ b/test/agent-cursor.test.ts@@ -1,19 +1,21 @@import { mkdtemp, readFile, rm } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";-import { afterEach, expect, test } from "vitest";+import { afterEach, expect, test, vi } from "vitest";import { McpDriver } from "../src/driver.ts";+import { VisualSettings } from "../src/native-visuals.ts";const dirs: string[] = [];afterEach(async () => { await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });-async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals" | "visuals-refused") {+async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals" | "visuals-refused" | "visuals-retry-discovery" | "visuals-retry-configuration") {const dir = await mkdtemp(path.join(tmpdir(), "agent-cursor-"));dirs.push(dir);const log = path.join(dir, "calls.jsonl");- const driver = new McpDriver({ command: process.execPath, args: [path.resolve("test/fixtures/agent-cursor-driver.mjs"), log, mode] });+ const settings = new VisualSettings();+ const driver = new McpDriver({ command: process.execPath, args: [path.resolve("test/fixtures/agent-cursor-driver.mjs"), log, mode] }, "0.33.2", settings);const calls = async () => (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line) as { name: string; arguments: Record<string, unknown> });- return { driver, calls };+ return { driver, calls, settings };}test("the driver makes the agent cursor jump and checks the setting before any other call", async () => {@@ -39,6 +41,26 @@ test("a native visual driver receives presentation settings without forcing a onexpect((await calls()).at(-1)).toMatchObject({ name: "present_agent_activity", arguments: { kind: "overview", generation: 1 } });} finally { await driver.close(); }});+test.each(["discovery", "configuration"] as const)("visuals and off preferences survive cancellation during first %s", async phase => {+ const { driver, calls, settings } = await start(`visuals-retry-${phase}`);+ const controller = new AbortController();+ const attempt = driver.call("list_windows", {}, controller.signal).catch(error => error);+ try {+ const waiting = phase === "discovery" ? "discovery_wait" : "set_agent_visuals";+ await vi.waitFor(async () => expect((await calls()).some(call => call.name === waiting)).toBe(true));+ controller.abort();+ expect(await attempt).toBeInstanceOf(Error);+ expect((await driver.call("list_windows", {})).isError).toBeFalsy();+ const frame = { target: { pid: 42, window_id: 101 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`, bounds: { x: 0, y: 0, width: 800, height: 600 } };+ expect(await driver.visuals(frame)).toMatchObject({ status: "queued" });+ await vi.waitFor(async () => expect((await calls()).at(-1)).toMatchObject({ name: "present_agent_activity", arguments: { kind: "overview" } }));+ settings.update({ preset: "off" });+ await vi.waitFor(async () => expect((await calls()).at(-1)).toMatchObject({ name: "set_agent_visuals", arguments: { preset: "off" } }));+ expect(await driver.visuals(frame)).toMatchObject({ status: "skipped" });+ expect((await calls()).filter(call => call.name === "list_windows")).toHaveLength(1);+ expect((await calls()).filter(call => call.name === "present_agent_activity")).toHaveLength(1);+ } finally { controller.abort(); await attempt; await driver.close(); }+});test("presentation configuration failure leaves input usable without replay", async () => {const { driver, calls } = await start("visuals-refused");try {diff --git a/test/fixtures/agent-cursor-driver.mjs b/test/fixtures/agent-cursor-driver.mjsindex 5ab810f..28439be 100644--- a/test/fixtures/agent-cursor-driver.mjs+++ b/test/fixtures/agent-cursor-driver.mjs@@ -1,4 +1,4 @@-import { appendFileSync } from "node:fs";+import { appendFileSync, existsSync, writeFileSync } from "node:fs";import { Server } from "@modelcontextprotocol/sdk/server/index.js";import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";@@ -8,9 +8,19 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotconst [log, mode] = process.argv.slice(2);let glide = 0;const server = new Server({ name: "cua-driver", version: "0.33.2" }, { capabilities: { tools: {} } });-server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: mode.startsWith("visuals") ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] }));+server.setRequestHandler(ListToolsRequestSchema, async () => {+ if (mode === "visuals-retry-discovery" && !existsSync(log)) {+ appendFileSync(log, JSON.stringify({ name: "discovery_wait", arguments: {} }) + "\n");+ await new Promise(() => {});+ }+ return { tools: mode.startsWith("visuals") ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] };+});server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, arguments: args = {} } }) => {appendFileSync(log, JSON.stringify({ name, arguments: args }) + "\n");+ if (name === "set_agent_visuals" && mode === "visuals-retry-configuration" && !existsSync(log + ".configured")) {+ writeFileSync(log + ".configured", "ready");+ await new Promise(() => {});+ }if (name === "set_agent_visuals" && mode === "visuals-refused") return { content: [], isError: true };if (name === "set_agent_cursor_motion") {if (mode === "refuses") return { content: [{ type: "text", text: "refused" }], isError: true };