Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Tests: Computer Use tool cards in the Pi TUI (LPT-856)
alexshpunt-clanker[bot]b632dbe118e6
diff --git a/test/computer-renderers.test.ts b/test/computer-renderers.test.tsnew file mode 100644index 0000000..2e6c86c--- /dev/null+++ b/test/computer-renderers.test.ts@@ -0,0 +1,105 @@+import { visibleWidth } from "@earendil-works/pi-tui";+import { expect, test } from "vitest";+import { computerActionRenderer, computerReadRenderer, typedRenderers } from "../src/computer-renderers.ts";++// Plain theme: styling is not the contract, the words and the layout are.+const theme = { fg: (_: unknown, text: string) => text, bold: (text: string) => text, underline: (text: string) => text };+const describe = (source: string) => source.includes("w1") ? { machine: "zinpc", title: "Untitled - Notepad" } : undefined;+const ctx = (isError = false) => ({ isError, args: {}, lastComponent: undefined });+const action = (data: Record<string, unknown>, isError = false) => ({ content: [{ type: "text", text: "PRIVATE_AGENT_TEXT" }], isError,+ details: { metadata: { semanticAction: { kind: "computer-action", ...data } } } });+const draw = (component: { render(width: number): string[] }, width = 80) => component.render(width);+const result = (tool: string, data: Record<string, unknown>, options: { expanded?: boolean; isPartial?: boolean } = {}, width = 80, isError = false) =>+ draw(typedRenderers(tool, describe).renderResult(action(data, isError) as never, { expanded: false, isPartial: false, ...options }, theme as never, ctx(isError) as never), width);++test("a delivered action says where it ran and what changed in the UI, not the receipt text", () => {+ const lines = result("computer_click", { ok: true, effect: "applied", status: "delivered", machine: "zinpc", script: { window: "computer:zinpc/w1ab" },+ successor: { diff: { kind: "ui", changed: ["1:a Edit value=1"], added: [], removed: [], counts: { changed: 1, added: 0, removed: 0 }, waited_ms: 180 } } });+ expect(lines[0]).toBe("✓ delivered · zinpc");+ expect(lines.join("\n")).toMatch(/1 changed.*180 ms/);+ expect(lines.join("\n")).toContain("~ 1:a Edit value=1");+ expect(lines.join("\n")).not.toContain("PRIVATE_AGENT_TEXT");+});++test("an action that changed nothing says so in a note", () => {+ const lines = result("computer_click", { ok: true, status: "delivered", machine: "zinpc",+ successor: { diff: { kind: "ui", changed: [], added: [], removed: [], counts: { changed: 0, added: 0, removed: 0 }, waited_ms: 300 } } });+ // The fixture's wait time is the one fact worth checking; the sentence around it is wording.+ expect(lines).toHaveLength(2);+ expect(lines[1]).toContain("300 ms");+});++test("unknown warns that the action may have happened; refused and failed give their one-line reason", () => {+ const unknown = result("computer_type", { ok: false, effect: "unknown", status: "unknown", machine: "zinpc", script: { reason: "Driver returned unknown for type_text. Observe again." } }, {}, 200);+ expect(unknown[0]).toBe("? unknown · zinpc");+ // The reason from the receipt, then one caution line that the action may have happened.+ expect(unknown).toHaveLength(3);+ expect(unknown[1]).toContain("Driver returned unknown for type_text");+ const refused = result("computer_click", { ok: false, effect: "not-applied", status: "refused", error: { message: "4:a is not a ref of this window's latest Read." } });+ expect(refused[0]).toBe("✗ refused");+ expect(refused[1]).toContain("4:a is not a ref");+ const failed = draw(typedRenderers("computer_click", describe).renderResult({ content: [{ type: "text", text: 'Validation failed for tool "computer_click":\n - x: must be number' }], isError: true } as never,+ { expanded: false, isPartial: false }, theme as never, ctx(true) as never), 100);+ expect(failed.join("\n")).toContain("✗ failed");+ expect(failed.join("\n")).toContain('Validation failed for tool "computer_click": x: must be number');+});++test("a resize that Windows settled within the allowed difference shows the measured size", () => {+ const lines = result("computer_window", { ok: true, status: "delivered", machine: "zinpc", state: { operation: "resize", mutation_attempted: true, postcondition_confirmed: true,+ requested_bounds: { width: 600, height: 450 }, observed: { bounds: { width: 601, height: 450 } } } });+ expect(lines[0]).toBe("✓ delivered · zinpc · resize · 601×450");+ expect(lines).toHaveLength(2);+ expect(lines[1]).toContain("601×450");+ expect(lines[1]).toContain("600×450");+});++test("while a call runs the card shows only the running state", () => {+ expect(result("computer_click", {}, { isPartial: true })).toEqual(["● running"]);+});++test("headings name the window by title, show literal input only by length, and drop options before clipping the target", () => {+ const heading = (args: Record<string, unknown>, width = 100) => draw(typedRenderers("computer_type", describe).renderCall(args, theme as never, { ...ctx(), expanded: false } as never), width);+ expect(heading({ window: "computer:zinpc/w1ab", target: "2:a", text: "PRIVATE_TYPED_TEXT", submit: true })[0]).toBe("computer_type “Untitled - Notepad” · 2:a · 18 chars · + Enter");+ const narrow = heading({ window: "computer:zinpc/w1ab", target: "2:a", text: "x", submit: true }, 40)[0];+ expect(visibleWidth(narrow)).toBeLessThanOrEqual(40);+ expect(narrow).toContain("computer_type “Untitled - Notepad”");+ const expanded = draw(typedRenderers("computer_type", describe).renderCall({ window: "computer:zinpc/w1ab", text: "PRIVATE_TYPED_TEXT", delay_ms: 0 }, theme as never, { ...ctx(), expanded: true } as never), 100).join("\n");+ expect(expanded).toContain("window=computer:zinpc/w1ab");+ expect(expanded).toContain("delay_ms=0");+ expect(expanded).not.toContain("PRIVATE_TYPED_TEXT");+});++test("app text is data: control sequences never reach the terminal and every line fits the width", () => {+ const lines = result("computer_click", { ok: false, effect: "not-applied", status: "refused", error: { message: "bad \u001b[2J target \u0007 测试 e\u0301 " + "long ".repeat(60) } }, {}, 24);+ expect(lines.join("\n")).not.toMatch(/[\u0000-\u0008\u001b]/);+ expect(lines.every(line => visibleWidth(line) <= 24)).toBe(true);+ expect(lines.join("\n")).toContain("测试");+});++test("a Read shows the listing or UI structure as rows, bounded, with the rest behind expand", () => {+ const elements = Array.from({ length: 9 }, (_, i) => ({ hint: `1:${"abcdefghi"[i]}`, role: "Button", name: `Button ${i}`, value: "PRIVATE_FIELD_VALUE", enabled: true }));+ const read = (expanded: boolean) => draw(computerReadRenderer(describe)({ content: [{ type: "text", text: "PRIVATE_AGENT_TEXT" }], details: { source: "computer:window/x", resolvedBy: "computer",+ computer: { status: "observed", machine: "zinpc", window_ref: "w1", agent_image: { output: { width: 500, height: 350 } }, ui: { collected: 9, truncated: false, elements } } } } as never,+ { expanded, isPartial: false }, theme as never, ctx() as never), 100);+ const compact = read(false);+ expect(compact[0]).toContain("✓ observed · zinpc");+ expect(compact[0]).toContain("image 500×350 · UI 9 elements");+ expect(compact).toHaveLength(1 + 6 + 1);+ expect(compact.at(-1)).toContain("3");+ expect(read(true).filter(line => line.includes("Button "))).toHaveLength(9);+ expect(compact.concat(read(true)).join("\n")).not.toMatch(/PRIVATE_/);+});++test("write, insert and delete renderers select only computer addresses and semantic computer results", () => {+ for (const tool of ["write", "insert", "delete"] as const) {+ const renderer = computerActionRenderer(tool, describe);+ expect(renderer.matches({ path: "computer:observation/owned" }, "call")).toBe(true);+ expect(renderer.matches({ path: "notes.txt" }, "call")).toBe(false);+ expect(renderer.matches({ path: ["computer:session"] }, "call")).toBe(false);+ expect(renderer.matches({ details: { metadata: { semanticAction: { kind: "computer-action" } } } }, "result")).toBe(true);+ for (const value of [undefined, { details: {} }, { details: { metadata: { semanticAction: { kind: "file-action" } } } }]) expect(renderer.matches(value, "result")).toBe(false);+ }+ const lines = draw(computerActionRenderer("write", describe).renderCall({ path: "computer:zinpc/o1ab", content: "PRIVATE_TYPED_TEXT" }, theme as never, { ...ctx(), expanded: false } as never));+ expect(lines[0]).toBe("write observation o1ab · zinpc");+ expect(lines.join("\n")).not.toContain("PRIVATE_TYPED_TEXT");+});diff --git a/test/fixtures/native-tui-extension.ts b/test/fixtures/native-tui-extension.tsnew file mode 100644index 0000000..a010034--- /dev/null+++ b/test/fixtures/native-tui-extension.ts@@ -0,0 +1,46 @@+import { Type } from "@earendil-works/pi-ai";+import { createCodemodeExtension, type ExtensionAPI } from "@earendil-works/pi-coding-agent";+import path from "node:path";+import { pathToFileURL } from "node:url";+import type { ComputerMode, ComputerResult, ComputerSession, WindowTarget } from "../../src/session.ts";++/**+ * Test-only: the real computer provider and typed tools over the synthetic ide-provider driver, plus fixture_control to+ * make the next native input unknown or to close a window. Used to look at the real Pi TUI; it never touches a desktop.+ */+export default async function fixture(pi: ExtensionAPI): Promise<void> {+ const source = path.resolve(import.meta.dirname, "../..");+ const load = (name: string) => import(pathToFileURL(path.join(source, "src", name)).href);+ const [{ registerComputerProvider }, { loadComputerConfig }, { McpDriver }, { ComputerSession }, { registerComputerTools }] =+ await Promise.all([load("provider.ts"), load("config.ts"), load("driver.ts"), load("session.ts"), load("computer-tools.ts")]);+ const Driver = McpDriver as typeof import("../../src/driver.ts").McpDriver;+ class FixtureDriver extends Driver {+ get platform() { return "windows" as const; }+ async identity(target: WindowTarget, _create: boolean, signal?: AbortSignal) {+ const result = await this.call("fixture_identity", target as unknown as Record<string, unknown>, signal);+ return result.structuredContent?.identity as string | undefined;+ }+ }+ const mode: ComputerMode = "yolo";+ let session: ComputerSession | undefined, driver: FixtureDriver | undefined;+ const getSession = async (cwd: string): Promise<ComputerSession> => {+ driver ??= new FixtureDriver(await loadComputerConfig(cwd));+ session ??= new ComputerSession(driver, () => mode);+ return session!;+ };+ const close = async (): Promise<ComputerResult> => session ? session.close() : { content: [{ type: "text", text: "No fixture runtime is running." }], details: { status: "closed", mode } };+ createCodemodeExtension({ mode: "on" })(pi);+ const provider = await registerComputerProvider(pi, getSession, close);+ registerComputerTools(pi, provider.resources);+ // Addresses carry a random per-run tag; a fixed one lets a scripted conversation name its windows.+ Object.defineProperty((provider.resources as unknown as { shorts: object }).shorts, "tag", { value: "tt" });+ pi.registerTool({+ name: "fixture_control", label: "Fixture control", description: "Make the synthetic driver's next input partial, or close its first window.",+ parameters: Type.Object({ operation: Type.Union([Type.Literal("partial"), Type.Literal("close_first"), Type.Literal("success")]) }),+ async execute(_id, { operation }, signal, _update, ctx) {+ await getSession(ctx.cwd);+ await driver!.call("fixture_control", { operation }, signal);+ return { content: [{ type: "text", text: `fixture ${operation}` }], details: {} };+ },+ });+}diff --git a/test/native-tui.integration.test.ts b/test/native-tui.integration.test.tsnew file mode 100644index 0000000..459ab7f--- /dev/null+++ b/test/native-tui.integration.test.ts@@ -0,0 +1,85 @@+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";+import path from "node:path";+import { assistantMessage, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";+import { expect, test } from "vitest";+import { IDE_EXTENSION, PI_COMMAND } from "./fixtures/pi-agent-ide.ts";++const root = path.resolve(import.meta.dirname, "..");+const call = (id: string, name: string, args: Record<string, unknown>) => assistantMessage([toolCall({ id, name, arguments: args })], { stopReason: "toolUse" });+const tools = ["read", "search", "write", "insert", "delete", "codemode", "fixture_control", "computer_open", "computer_click", "computer_type", "computer_window"];+const w = "computer:local/w1tt";++/** One scripted conversation through the real Pi TUI over the synthetic driver; returns the final screen without trailing blanks. */+async function screenOf(name: string, conversation: ReturnType<typeof call>[], size: { cols: number; rows: number }, expanded: boolean) {+ await mkdir(path.join(root, ".tmp/lpt-856"), { recursive: true });+ const cwd = await mkdtemp(path.join(root, ".tmp/lpt-856/pi-tui-"));+ try {+ await mkdir(path.join(cwd, ".pi"));+ await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath, args: [path.join(root, "test/fixtures/ide-provider-driver.mjs"), path.join(cwd, "calls.jsonl")] }));+ const result = await new PiIntegrationTest({+ testName: name, artifactsDir: testArtifactsDir(import.meta.filename), cwd, isolateUserResources: true, rawMode: false, piCommand: PI_COMMAND, tuiSize: size,+ // The harness expands tool output unless it runs live; compact is the user's normal view.+ environment: expanded ? {} : { PI_INTEGRATION_TEST_LIVE: "1" },+ extensions: [IDE_EXTENSION, path.join(root, "test/fixtures/native-tui-extension.ts")], tools, timeoutMs: 120_000,+ conversation: [...conversation, assistantMessage([text("done")])],+ }).run("Look at the synthetic fixture.");+ return result.tuiRenderedOutput.split("\n").map(line => line.trimEnd()).join("\n").replace(/\n{3,}/g, "\n\n");+ } finally { await rm(cwd, { recursive: true, force: true }); }+}++// Contract: every Computer Use call shows a title line and one status line (delivered, unknown, refused), a few+// scannable rows, and never the agent's receipt text, typed input or a guide dump. Layout holds at 60 columns.+const flow = [+ call("c1", "computer_open", { at: "computer:machine/local", app: "Notepad" }),+ call("c2", "search", { path: "computer:machine/local/windows", query: "Owned fixture" }),+ call("c3", "read", { path: w }),+ // Points are pixels of the latest image, so these steps do not depend on how many snapshots came before.+ call("c5", "computer_click", { window: w, x: 240, y: 180 }),+ call("c6", "read", { path: w }),+ call("c7", "fixture_control", { operation: "partial" }),+ call("c8", "computer_click", { window: w, x: 240, y: 180 }),+ call("c9", "fixture_control", { operation: "close_first" }),+ call("c10", "computer_click", { window: w, x: 240, y: 180 }),+ // Not a valid call (typing takes a target, not a point): Pi's own validation refuses it before the tool runs.+ call("c10b", "computer_type", { window: w, text: "PRIVATE_TYPED_TEXT", delay_ms: 0, x: 240, y: 180 }),+ call("c11", "write", { path: "computer:local/o1tt", content: "PRIVATE_WRITTEN_TEXT" }),+ call("c12", "delete", { path: w }),+];++for (const [label, size] of [["normal", { cols: 160, rows: 100 }], ["narrow", { cols: 60, rows: 140 }]] as const) test(`${label} terminal: observation, UI rows and every action outcome read at a glance`, async () => {+ const screen = await screenOf(`states-${label}`, flow, size, false);+ // A narrow terminal wraps lines, so facts are compared with their whitespace collapsed.+ const flat = screen.replace(/\s+/g, " ");+ // Observation and discovery.+ expect(flat).toContain("computer_open Notepad · local ✓ observed · local · image 500×350 · UI 2 elements 1:a Edit “Fixture text field”");+ expect(flat).toContain("✓ listed · local · 2 windows Notepad “Unowned fixture” 1000×700 Notepad “Owned fixture 91” 1000×700 yours");+ // Actions: delivered, unknown (may have been applied), refused (nothing sent), failed before the tool ran.+ expect(flat).toContain("computer_click “Owned fixture 91” · 240,180 ✓ delivered · local");+ expect(flat).toContain("? unknown · local Driver returned unknown for click");+ expect(flat).toContain("✗ refused · local Window closed");+ expect(flat).toContain("computer_type “Owned fixture 91” · 240,180 · 18 chars ✗ failed Validation failed for tool \"computer_type\"");+ // What the agent sends and gets back stays out of the human view.+ for (const hidden of ["PRIVATE_TYPED_TEXT", "PRIVATE_WRITTEN_TEXT", "Computer delivered (yolo)", "# Guide:", "system-result", "Initial fixture value", "\u001b"]) expect(flat, hidden).not.toContain(hidden);+ expect(Math.max(...screen.split("\n").map(line => [...line].length))).toBeLessThanOrEqual(size.cols);+}, 180_000);++test("expanded: exact arguments and details appear on demand, still without typed text", async () => {+ const screen = await screenOf("states-expanded", flow, { cols: 160, rows: 140 }, true);+ expect(screen).toMatch(/computer_click “Owned fixture 91” · 240,180\n .*window=computer:local\/w1tt · x=240 · y=180/);+ // Details carry the receipt's own codes and values.+ expect(screen).toContain("COMPUTER_");+ expect(screen).toContain("unverified");+ for (const hidden of ["PRIVATE_TYPED_TEXT", "PRIVATE_WRITTEN_TEXT", "Computer delivered (yolo)", "system-result"]) expect(screen, hidden).not.toContain(hidden);+}, 180_000);++// Pi's Codemode draws its own activity rows for nested calls and has no hook for a tool to restyle them; they stay as they are.+test("a nested Codemode script keeps Pi's activity rows and returns the receipts to the script", async () => {+ const screen = await screenOf("nested", [call("n1", "codemode", { code: `const open = await tools.computer_open({ at: "computer:machine/local", app: "Notepad" });+const view = await tools.read({ path: open.window });+const click = await tools.computer_click({ window: open.window, x: 240, y: 180 });+text(JSON.stringify({ open: open.outcome, click: click.outcome }));` })], { cols: 160, rows: 100 }, false);+ expect(screen).toMatch(/✓ computer_open /);+ expect(screen).toMatch(/✓ read /);+ expect(screen).toMatch(/✓ computer_click /);+ expect(screen).toContain('{"open":"observed","click":"delivered"}');+}, 180_000);