Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

fix: preserve native visuals across startup retries

alexshpunt-clanker[bot]cfc6bb497a25
diff --git a/src/driver.ts b/src/driver.ts
index 08bd346..9a81903 100644
--- a/src/driver.ts
+++ b/src/driver.ts
@@ -245,9 +245,10 @@ export class McpDriver implements Driver {
this.nativeVisualsAvailable = false;
}
} else await this.jumpCursor(client, signal);
+ signal?.throwIfAborted();
return client;
} catch (error) {
- await this.close();
+ await this.disconnect();
throw new Error(`Driver startup failed: ${error instanceof Error ? error.message : error}\n${this.stderr}`);
}
}
@@ -379,6 +380,11 @@ export class McpDriver implements Driver {
/** EOF owns shutdown; the SDK escalates only for this child if it does not exit. */
async close(): Promise<void> {
this.presentation.close();
+ await this.disconnect();
+ }
+
+ /** Startup cleanup is retryable; only final close disposes presentation and its subscriptions. */
+ private async disconnect(): Promise<void> {
this.nativeVisualsAvailable = false;
const client = this.client;
const transport = this.transport;
diff --git a/test/agent-cursor.test.ts b/test/agent-cursor.test.ts
index d781c8e..b1bbc87 100644
--- a/test/agent-cursor.test.ts
+++ b/test/agent-cursor.test.ts
@@ -1,19 +1,21 @@
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
-import { afterEach, expect, test } from "vitest";
+import { afterEach, expect, test, vi } from "vitest";
import { McpDriver } from "../src/driver.ts";
+import { VisualSettings } from "../src/native-visuals.ts";
const dirs: string[] = [];
afterEach(async () => { await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });
-async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals" | "visuals-refused") {
+async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals" | "visuals-refused" | "visuals-retry-discovery" | "visuals-retry-configuration") {
const dir = await mkdtemp(path.join(tmpdir(), "agent-cursor-"));
dirs.push(dir);
const log = path.join(dir, "calls.jsonl");
- const driver = new McpDriver({ command: process.execPath, args: [path.resolve("test/fixtures/agent-cursor-driver.mjs"), log, mode] });
+ const settings = new VisualSettings();
+ const driver = new McpDriver({ command: process.execPath, args: [path.resolve("test/fixtures/agent-cursor-driver.mjs"), log, mode] }, "0.33.2", settings);
const calls = async () => (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line) as { name: string; arguments: Record<string, unknown> });
- return { driver, calls };
+ return { driver, calls, settings };
}
test("the driver makes the agent cursor jump and checks the setting before any other call", async () => {
@@ -39,6 +41,26 @@ test("a native visual driver receives presentation settings without forcing a on
expect((await calls()).at(-1)).toMatchObject({ name: "present_agent_activity", arguments: { kind: "overview", generation: 1 } });
} finally { await driver.close(); }
});
+test.each(["discovery", "configuration"] as const)("visuals and off preferences survive cancellation during first %s", async phase => {
+ const { driver, calls, settings } = await start(`visuals-retry-${phase}`);
+ const controller = new AbortController();
+ const attempt = driver.call("list_windows", {}, controller.signal).catch(error => error);
+ try {
+ const waiting = phase === "discovery" ? "discovery_wait" : "set_agent_visuals";
+ await vi.waitFor(async () => expect((await calls()).some(call => call.name === waiting)).toBe(true));
+ controller.abort();
+ expect(await attempt).toBeInstanceOf(Error);
+ expect((await driver.call("list_windows", {})).isError).toBeFalsy();
+ const frame = { target: { pid: 42, window_id: 101 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`, bounds: { x: 0, y: 0, width: 800, height: 600 } };
+ expect(await driver.visuals(frame)).toMatchObject({ status: "queued" });
+ await vi.waitFor(async () => expect((await calls()).at(-1)).toMatchObject({ name: "present_agent_activity", arguments: { kind: "overview" } }));
+ settings.update({ preset: "off" });
+ await vi.waitFor(async () => expect((await calls()).at(-1)).toMatchObject({ name: "set_agent_visuals", arguments: { preset: "off" } }));
+ expect(await driver.visuals(frame)).toMatchObject({ status: "skipped" });
+ expect((await calls()).filter(call => call.name === "list_windows")).toHaveLength(1);
+ expect((await calls()).filter(call => call.name === "present_agent_activity")).toHaveLength(1);
+ } finally { controller.abort(); await attempt; await driver.close(); }
+});
test("presentation configuration failure leaves input usable without replay", async () => {
const { driver, calls } = await start("visuals-refused");
try {
diff --git a/test/fixtures/agent-cursor-driver.mjs b/test/fixtures/agent-cursor-driver.mjs
index 5ab810f..28439be 100644
--- a/test/fixtures/agent-cursor-driver.mjs
+++ b/test/fixtures/agent-cursor-driver.mjs
@@ -1,4 +1,4 @@
-import { appendFileSync } from "node:fs";
+import { appendFileSync, existsSync, writeFileSync } from "node:fs";
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
@@ -8,9 +8,19 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprot
const [log, mode] = process.argv.slice(2);
let glide = 0;
const server = new Server({ name: "cua-driver", version: "0.33.2" }, { capabilities: { tools: {} } });
-server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: mode.startsWith("visuals") ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] }));
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+ if (mode === "visuals-retry-discovery" && !existsSync(log)) {
+ appendFileSync(log, JSON.stringify({ name: "discovery_wait", arguments: {} }) + "\n");
+ await new Promise(() => {});
+ }
+ return { tools: mode.startsWith("visuals") ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] };
+});
server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, arguments: args = {} } }) => {
appendFileSync(log, JSON.stringify({ name, arguments: args }) + "\n");
+ if (name === "set_agent_visuals" && mode === "visuals-retry-configuration" && !existsSync(log + ".configured")) {
+ writeFileSync(log + ".configured", "ready");
+ await new Promise(() => {});
+ }
if (name === "set_agent_visuals" && mode === "visuals-refused") return { content: [], isError: true };
if (name === "set_agent_cursor_motion") {
if (mode === "refuses") return { content: [{ type: "text", text: "refused" }], isError: true };