Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Tests: Computer Use tool cards in the Pi TUI (LPT-856)

alexshpunt-clanker[bot]b632dbe118e6
diff --git a/test/computer-renderers.test.ts b/test/computer-renderers.test.ts
new file mode 100644
index 0000000..2e6c86c
--- /dev/null
+++ b/test/computer-renderers.test.ts
@@ -0,0 +1,105 @@
+import { visibleWidth } from "@earendil-works/pi-tui";
+import { expect, test } from "vitest";
+import { computerActionRenderer, computerReadRenderer, typedRenderers } from "../src/computer-renderers.ts";
+
+// Plain theme: styling is not the contract, the words and the layout are.
+const theme = { fg: (_: unknown, text: string) => text, bold: (text: string) => text, underline: (text: string) => text };
+const describe = (source: string) => source.includes("w1") ? { machine: "zinpc", title: "Untitled - Notepad" } : undefined;
+const ctx = (isError = false) => ({ isError, args: {}, lastComponent: undefined });
+const action = (data: Record<string, unknown>, isError = false) => ({ content: [{ type: "text", text: "PRIVATE_AGENT_TEXT" }], isError,
+ details: { metadata: { semanticAction: { kind: "computer-action", ...data } } } });
+const draw = (component: { render(width: number): string[] }, width = 80) => component.render(width);
+const result = (tool: string, data: Record<string, unknown>, options: { expanded?: boolean; isPartial?: boolean } = {}, width = 80, isError = false) =>
+ draw(typedRenderers(tool, describe).renderResult(action(data, isError) as never, { expanded: false, isPartial: false, ...options }, theme as never, ctx(isError) as never), width);
+
+test("a delivered action says where it ran and what changed in the UI, not the receipt text", () => {
+ const lines = result("computer_click", { ok: true, effect: "applied", status: "delivered", machine: "zinpc", script: { window: "computer:zinpc/w1ab" },
+ successor: { diff: { kind: "ui", changed: ["1:a Edit value=1"], added: [], removed: [], counts: { changed: 1, added: 0, removed: 0 }, waited_ms: 180 } } });
+ expect(lines[0]).toBe("✓ delivered · zinpc");
+ expect(lines.join("\n")).toMatch(/1 changed.*180 ms/);
+ expect(lines.join("\n")).toContain("~ 1:a Edit value=1");
+ expect(lines.join("\n")).not.toContain("PRIVATE_AGENT_TEXT");
+});
+
+test("an action that changed nothing says so in a note", () => {
+ const lines = result("computer_click", { ok: true, status: "delivered", machine: "zinpc",
+ successor: { diff: { kind: "ui", changed: [], added: [], removed: [], counts: { changed: 0, added: 0, removed: 0 }, waited_ms: 300 } } });
+ // The fixture's wait time is the one fact worth checking; the sentence around it is wording.
+ expect(lines).toHaveLength(2);
+ expect(lines[1]).toContain("300 ms");
+});
+
+test("unknown warns that the action may have happened; refused and failed give their one-line reason", () => {
+ const unknown = result("computer_type", { ok: false, effect: "unknown", status: "unknown", machine: "zinpc", script: { reason: "Driver returned unknown for type_text. Observe again." } }, {}, 200);
+ expect(unknown[0]).toBe("? unknown · zinpc");
+ // The reason from the receipt, then one caution line that the action may have happened.
+ expect(unknown).toHaveLength(3);
+ expect(unknown[1]).toContain("Driver returned unknown for type_text");
+ const refused = result("computer_click", { ok: false, effect: "not-applied", status: "refused", error: { message: "4:a is not a ref of this window's latest Read." } });
+ expect(refused[0]).toBe("✗ refused");
+ expect(refused[1]).toContain("4:a is not a ref");
+ const failed = draw(typedRenderers("computer_click", describe).renderResult({ content: [{ type: "text", text: 'Validation failed for tool "computer_click":\n - x: must be number' }], isError: true } as never,
+ { expanded: false, isPartial: false }, theme as never, ctx(true) as never), 100);
+ expect(failed.join("\n")).toContain("✗ failed");
+ expect(failed.join("\n")).toContain('Validation failed for tool "computer_click": x: must be number');
+});
+
+test("a resize that Windows settled within the allowed difference shows the measured size", () => {
+ const lines = result("computer_window", { ok: true, status: "delivered", machine: "zinpc", state: { operation: "resize", mutation_attempted: true, postcondition_confirmed: true,
+ requested_bounds: { width: 600, height: 450 }, observed: { bounds: { width: 601, height: 450 } } } });
+ expect(lines[0]).toBe("✓ delivered · zinpc · resize · 601×450");
+ expect(lines).toHaveLength(2);
+ expect(lines[1]).toContain("601×450");
+ expect(lines[1]).toContain("600×450");
+});
+
+test("while a call runs the card shows only the running state", () => {
+ expect(result("computer_click", {}, { isPartial: true })).toEqual(["● running"]);
+});
+
+test("headings name the window by title, show literal input only by length, and drop options before clipping the target", () => {
+ const heading = (args: Record<string, unknown>, width = 100) => draw(typedRenderers("computer_type", describe).renderCall(args, theme as never, { ...ctx(), expanded: false } as never), width);
+ expect(heading({ window: "computer:zinpc/w1ab", target: "2:a", text: "PRIVATE_TYPED_TEXT", submit: true })[0]).toBe("computer_type “Untitled - Notepad” · 2:a · 18 chars · + Enter");
+ const narrow = heading({ window: "computer:zinpc/w1ab", target: "2:a", text: "x", submit: true }, 40)[0];
+ expect(visibleWidth(narrow)).toBeLessThanOrEqual(40);
+ expect(narrow).toContain("computer_type “Untitled - Notepad”");
+ const expanded = draw(typedRenderers("computer_type", describe).renderCall({ window: "computer:zinpc/w1ab", text: "PRIVATE_TYPED_TEXT", delay_ms: 0 }, theme as never, { ...ctx(), expanded: true } as never), 100).join("\n");
+ expect(expanded).toContain("window=computer:zinpc/w1ab");
+ expect(expanded).toContain("delay_ms=0");
+ expect(expanded).not.toContain("PRIVATE_TYPED_TEXT");
+});
+
+test("app text is data: control sequences never reach the terminal and every line fits the width", () => {
+ const lines = result("computer_click", { ok: false, effect: "not-applied", status: "refused", error: { message: "bad \u001b[2J target \u0007 测试 e\u0301 " + "long ".repeat(60) } }, {}, 24);
+ expect(lines.join("\n")).not.toMatch(/[\u0000-\u0008\u001b]/);
+ expect(lines.every(line => visibleWidth(line) <= 24)).toBe(true);
+ expect(lines.join("\n")).toContain("测试");
+});
+
+test("a Read shows the listing or UI structure as rows, bounded, with the rest behind expand", () => {
+ const elements = Array.from({ length: 9 }, (_, i) => ({ hint: `1:${"abcdefghi"[i]}`, role: "Button", name: `Button ${i}`, value: "PRIVATE_FIELD_VALUE", enabled: true }));
+ const read = (expanded: boolean) => draw(computerReadRenderer(describe)({ content: [{ type: "text", text: "PRIVATE_AGENT_TEXT" }], details: { source: "computer:window/x", resolvedBy: "computer",
+ computer: { status: "observed", machine: "zinpc", window_ref: "w1", agent_image: { output: { width: 500, height: 350 } }, ui: { collected: 9, truncated: false, elements } } } } as never,
+ { expanded, isPartial: false }, theme as never, ctx() as never), 100);
+ const compact = read(false);
+ expect(compact[0]).toContain("✓ observed · zinpc");
+ expect(compact[0]).toContain("image 500×350 · UI 9 elements");
+ expect(compact).toHaveLength(1 + 6 + 1);
+ expect(compact.at(-1)).toContain("3");
+ expect(read(true).filter(line => line.includes("Button "))).toHaveLength(9);
+ expect(compact.concat(read(true)).join("\n")).not.toMatch(/PRIVATE_/);
+});
+
+test("write, insert and delete renderers select only computer addresses and semantic computer results", () => {
+ for (const tool of ["write", "insert", "delete"] as const) {
+ const renderer = computerActionRenderer(tool, describe);
+ expect(renderer.matches({ path: "computer:observation/owned" }, "call")).toBe(true);
+ expect(renderer.matches({ path: "notes.txt" }, "call")).toBe(false);
+ expect(renderer.matches({ path: ["computer:session"] }, "call")).toBe(false);
+ expect(renderer.matches({ details: { metadata: { semanticAction: { kind: "computer-action" } } } }, "result")).toBe(true);
+ for (const value of [undefined, { details: {} }, { details: { metadata: { semanticAction: { kind: "file-action" } } } }]) expect(renderer.matches(value, "result")).toBe(false);
+ }
+ const lines = draw(computerActionRenderer("write", describe).renderCall({ path: "computer:zinpc/o1ab", content: "PRIVATE_TYPED_TEXT" }, theme as never, { ...ctx(), expanded: false } as never));
+ expect(lines[0]).toBe("write observation o1ab · zinpc");
+ expect(lines.join("\n")).not.toContain("PRIVATE_TYPED_TEXT");
+});
diff --git a/test/fixtures/native-tui-extension.ts b/test/fixtures/native-tui-extension.ts
new file mode 100644
index 0000000..a010034
--- /dev/null
+++ b/test/fixtures/native-tui-extension.ts
@@ -0,0 +1,46 @@
+import { Type } from "@earendil-works/pi-ai";
+import { createCodemodeExtension, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import type { ComputerMode, ComputerResult, ComputerSession, WindowTarget } from "../../src/session.ts";
+
+/**
+ * Test-only: the real computer provider and typed tools over the synthetic ide-provider driver, plus fixture_control to
+ * make the next native input unknown or to close a window. Used to look at the real Pi TUI; it never touches a desktop.
+ */
+export default async function fixture(pi: ExtensionAPI): Promise<void> {
+ const source = path.resolve(import.meta.dirname, "../..");
+ const load = (name: string) => import(pathToFileURL(path.join(source, "src", name)).href);
+ const [{ registerComputerProvider }, { loadComputerConfig }, { McpDriver }, { ComputerSession }, { registerComputerTools }] =
+ await Promise.all([load("provider.ts"), load("config.ts"), load("driver.ts"), load("session.ts"), load("computer-tools.ts")]);
+ const Driver = McpDriver as typeof import("../../src/driver.ts").McpDriver;
+ class FixtureDriver extends Driver {
+ get platform() { return "windows" as const; }
+ async identity(target: WindowTarget, _create: boolean, signal?: AbortSignal) {
+ const result = await this.call("fixture_identity", target as unknown as Record<string, unknown>, signal);
+ return result.structuredContent?.identity as string | undefined;
+ }
+ }
+ const mode: ComputerMode = "yolo";
+ let session: ComputerSession | undefined, driver: FixtureDriver | undefined;
+ const getSession = async (cwd: string): Promise<ComputerSession> => {
+ driver ??= new FixtureDriver(await loadComputerConfig(cwd));
+ session ??= new ComputerSession(driver, () => mode);
+ return session!;
+ };
+ const close = async (): Promise<ComputerResult> => session ? session.close() : { content: [{ type: "text", text: "No fixture runtime is running." }], details: { status: "closed", mode } };
+ createCodemodeExtension({ mode: "on" })(pi);
+ const provider = await registerComputerProvider(pi, getSession, close);
+ registerComputerTools(pi, provider.resources);
+ // Addresses carry a random per-run tag; a fixed one lets a scripted conversation name its windows.
+ Object.defineProperty((provider.resources as unknown as { shorts: object }).shorts, "tag", { value: "tt" });
+ pi.registerTool({
+ name: "fixture_control", label: "Fixture control", description: "Make the synthetic driver's next input partial, or close its first window.",
+ parameters: Type.Object({ operation: Type.Union([Type.Literal("partial"), Type.Literal("close_first"), Type.Literal("success")]) }),
+ async execute(_id, { operation }, signal, _update, ctx) {
+ await getSession(ctx.cwd);
+ await driver!.call("fixture_control", { operation }, signal);
+ return { content: [{ type: "text", text: `fixture ${operation}` }], details: {} };
+ },
+ });
+}
diff --git a/test/native-tui.integration.test.ts b/test/native-tui.integration.test.ts
new file mode 100644
index 0000000..459ab7f
--- /dev/null
+++ b/test/native-tui.integration.test.ts
@@ -0,0 +1,85 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { assistantMessage, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";
+import { expect, test } from "vitest";
+import { IDE_EXTENSION, PI_COMMAND } from "./fixtures/pi-agent-ide.ts";
+
+const root = path.resolve(import.meta.dirname, "..");
+const call = (id: string, name: string, args: Record<string, unknown>) => assistantMessage([toolCall({ id, name, arguments: args })], { stopReason: "toolUse" });
+const tools = ["read", "search", "write", "insert", "delete", "codemode", "fixture_control", "computer_open", "computer_click", "computer_type", "computer_window"];
+const w = "computer:local/w1tt";
+
+/** One scripted conversation through the real Pi TUI over the synthetic driver; returns the final screen without trailing blanks. */
+async function screenOf(name: string, conversation: ReturnType<typeof call>[], size: { cols: number; rows: number }, expanded: boolean) {
+ await mkdir(path.join(root, ".tmp/lpt-856"), { recursive: true });
+ const cwd = await mkdtemp(path.join(root, ".tmp/lpt-856/pi-tui-"));
+ try {
+ await mkdir(path.join(cwd, ".pi"));
+ await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath, args: [path.join(root, "test/fixtures/ide-provider-driver.mjs"), path.join(cwd, "calls.jsonl")] }));
+ const result = await new PiIntegrationTest({
+ testName: name, artifactsDir: testArtifactsDir(import.meta.filename), cwd, isolateUserResources: true, rawMode: false, piCommand: PI_COMMAND, tuiSize: size,
+ // The harness expands tool output unless it runs live; compact is the user's normal view.
+ environment: expanded ? {} : { PI_INTEGRATION_TEST_LIVE: "1" },
+ extensions: [IDE_EXTENSION, path.join(root, "test/fixtures/native-tui-extension.ts")], tools, timeoutMs: 120_000,
+ conversation: [...conversation, assistantMessage([text("done")])],
+ }).run("Look at the synthetic fixture.");
+ return result.tuiRenderedOutput.split("\n").map(line => line.trimEnd()).join("\n").replace(/\n{3,}/g, "\n\n");
+ } finally { await rm(cwd, { recursive: true, force: true }); }
+}
+
+// Contract: every Computer Use call shows a title line and one status line (delivered, unknown, refused), a few
+// scannable rows, and never the agent's receipt text, typed input or a guide dump. Layout holds at 60 columns.
+const flow = [
+ call("c1", "computer_open", { at: "computer:machine/local", app: "Notepad" }),
+ call("c2", "search", { path: "computer:machine/local/windows", query: "Owned fixture" }),
+ call("c3", "read", { path: w }),
+ // Points are pixels of the latest image, so these steps do not depend on how many snapshots came before.
+ call("c5", "computer_click", { window: w, x: 240, y: 180 }),
+ call("c6", "read", { path: w }),
+ call("c7", "fixture_control", { operation: "partial" }),
+ call("c8", "computer_click", { window: w, x: 240, y: 180 }),
+ call("c9", "fixture_control", { operation: "close_first" }),
+ call("c10", "computer_click", { window: w, x: 240, y: 180 }),
+ // Not a valid call (typing takes a target, not a point): Pi's own validation refuses it before the tool runs.
+ call("c10b", "computer_type", { window: w, text: "PRIVATE_TYPED_TEXT", delay_ms: 0, x: 240, y: 180 }),
+ call("c11", "write", { path: "computer:local/o1tt", content: "PRIVATE_WRITTEN_TEXT" }),
+ call("c12", "delete", { path: w }),
+];
+
+for (const [label, size] of [["normal", { cols: 160, rows: 100 }], ["narrow", { cols: 60, rows: 140 }]] as const) test(`${label} terminal: observation, UI rows and every action outcome read at a glance`, async () => {
+ const screen = await screenOf(`states-${label}`, flow, size, false);
+ // A narrow terminal wraps lines, so facts are compared with their whitespace collapsed.
+ const flat = screen.replace(/\s+/g, " ");
+ // Observation and discovery.
+ expect(flat).toContain("computer_open Notepad · local ✓ observed · local · image 500×350 · UI 2 elements 1:a Edit “Fixture text field”");
+ expect(flat).toContain("✓ listed · local · 2 windows Notepad “Unowned fixture” 1000×700 Notepad “Owned fixture 91” 1000×700 yours");
+ // Actions: delivered, unknown (may have been applied), refused (nothing sent), failed before the tool ran.
+ expect(flat).toContain("computer_click “Owned fixture 91” · 240,180 ✓ delivered · local");
+ expect(flat).toContain("? unknown · local Driver returned unknown for click");
+ expect(flat).toContain("✗ refused · local Window closed");
+ expect(flat).toContain("computer_type “Owned fixture 91” · 240,180 · 18 chars ✗ failed Validation failed for tool \"computer_type\"");
+ // What the agent sends and gets back stays out of the human view.
+ for (const hidden of ["PRIVATE_TYPED_TEXT", "PRIVATE_WRITTEN_TEXT", "Computer delivered (yolo)", "# Guide:", "system-result", "Initial fixture value", "\u001b"]) expect(flat, hidden).not.toContain(hidden);
+ expect(Math.max(...screen.split("\n").map(line => [...line].length))).toBeLessThanOrEqual(size.cols);
+}, 180_000);
+
+test("expanded: exact arguments and details appear on demand, still without typed text", async () => {
+ const screen = await screenOf("states-expanded", flow, { cols: 160, rows: 140 }, true);
+ expect(screen).toMatch(/computer_click “Owned fixture 91” · 240,180\n .*window=computer:local\/w1tt · x=240 · y=180/);
+ // Details carry the receipt's own codes and values.
+ expect(screen).toContain("COMPUTER_");
+ expect(screen).toContain("unverified");
+ for (const hidden of ["PRIVATE_TYPED_TEXT", "PRIVATE_WRITTEN_TEXT", "Computer delivered (yolo)", "system-result"]) expect(screen, hidden).not.toContain(hidden);
+}, 180_000);
+
+// Pi's Codemode draws its own activity rows for nested calls and has no hook for a tool to restyle them; they stay as they are.
+test("a nested Codemode script keeps Pi's activity rows and returns the receipts to the script", async () => {
+ const screen = await screenOf("nested", [call("n1", "codemode", { code: `const open = await tools.computer_open({ at: "computer:machine/local", app: "Notepad" });
+const view = await tools.read({ path: open.window });
+const click = await tools.computer_click({ window: open.window, x: 240, y: 180 });
+text(JSON.stringify({ open: open.outcome, click: click.outcome }));` })], { cols: 160, rows: 100 }, false);
+ expect(screen).toMatch(/✓ computer_open /);
+ expect(screen).toMatch(/✓ read /);
+ expect(screen).toMatch(/✓ computer_click /);
+ expect(screen).toContain('{"open":"observed","click":"delivered"}');
+}, 180_000);