Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: exercise visual preferences in a real Pi process
alexshpunt-clanker[bot]5556d7a6a6f6
diff --git a/test/fixtures/visual-commands-extension.ts b/test/fixtures/visual-commands-extension.tsnew file mode 100644index 0000000..5fcc654--- /dev/null+++ b/test/fixtures/visual-commands-extension.ts@@ -0,0 +1,24 @@+import { writeFile } from "node:fs/promises";+import path from "node:path";+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";+import { CuaImageSettings } from "../../src/cua-image-settings.ts";+import { registerImageSettingsCommand } from "../../src/image-settings-command.ts";+import { VisualSettings } from "../../src/native-visuals.ts";+import { registerVisualSettingsCommand } from "../../src/visual-settings-command.ts";++/** Real Pi owns command dispatch, session entries and tree navigation; this fixture only reads the outcome. */+export default function(pi: ExtensionAPI): void {+ const visuals = new VisualSettings(); const images = new CuaImageSettings();+ registerVisualSettingsCommand(pi, visuals); registerImageSettingsCommand(pi, images);+ pi.registerCommand("visual-probe", {+ description: "Save the current test preferences.",+ handler: async (_args, ctx) => {+ await writeFile(path.join(ctx.cwd, "preferences.json"), JSON.stringify({ visuals: visuals.current(), images: images.settings(),+ branch: ctx.sessionManager.getBranch() }));+ },+ });+ pi.registerCommand("visual-navigate", {+ description: "Navigate the test session without inference.",+ handler: async (target, ctx) => { await ctx.navigateTree(target.trim(), { summarize: false }); },+ });+}diff --git a/test/visual-commands.integration.test.ts b/test/visual-commands.integration.test.tsnew file mode 100644index 0000000..447c471--- /dev/null+++ b/test/visual-commands.integration.test.ts@@ -0,0 +1,45 @@+import { RpcClient } from "@earendil-works/pi-coding-agent";+import { mkdtemp, readFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { fileURLToPath } from "node:url";+import { expect, test } from "vitest";++test("human visual commands persist preferences and restore the selected Pi branch without model work", async () => {+ const cwd = await mkdtemp(path.join(tmpdir(), "computer-visual-commands-"));+ const client = new RpcClient({ cwd,+ cliPath: path.join(path.dirname(fileURLToPath(import.meta.resolve("@earendil-works/pi-coding-agent"))), "cli.js"),+ env: { PI_CODING_AGENT_DIR: path.join(cwd, "agent"), PI_OFFLINE: "1" },+ args: ["--no-session", "--no-extensions", "--no-mcp", "--no-skills", "--no-context-files", "--approve",+ "--extension", path.resolve("test/fixtures/visual-commands-extension.ts")],+ });+ const probe = async () => {+ expect(await client.prompt("/visual-probe")).toBe("handled");+ return JSON.parse(await readFile(path.join(cwd, "preferences.json"), "utf8")) as {+ visuals: { preset: string; previews: boolean; keyboard: boolean; quality: string };+ images: { maxLongEdge: number; hints: boolean };+ };+ };+ try {+ await client.start();+ const commands = await client.getCommands();+ expect(commands.filter(command => command.name === "computer-visuals")).toHaveLength(1);+ expect(commands.filter(command => command.name === "computer-images")).toHaveLength(1);+ expect(await client.prompt("/computer-visuals subtle")).toBe("handled");+ const saved = (await client.getEntries()).entries.filter(entry => entry.type === "custom" && entry.customType === "computer-use-visuals");+ expect(saved).toHaveLength(1);+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false });+ expect(await client.prompt('/computer-visuals {"preset":"full","keyboard":false,"quality":"high"}')).toBe("handled");+ expect((await probe()).visuals).toMatchObject({ preset: "full", keyboard: false, quality: "high" });+ expect(await client.prompt(`/visual-navigate ${saved[0].id}`)).toBe("handled");+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false, keyboard: true, quality: "balanced" });+ expect(await client.prompt("/computer-images cap 960")).toBe("handled");+ expect(await client.prompt("/computer-images hints off")).toBe("handled");+ expect((await probe()).images).toMatchObject({ maxLongEdge: 960, hints: false });+ expect(await client.prompt('/computer-visuals {"constructor":false}')).toBe("handled");+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false });+ expect((await client.getEntries()).entries.filter(entry => entry.type === "custom" && entry.customType === "computer-use-visuals")).toHaveLength(2);+ expect((await client.getMessages()).filter(message => message.role === "assistant")).toHaveLength(0);+ expect(client.getStderr()).toBe("");+ } finally { await client.stop(); await rm(cwd, { recursive: true, force: true }); }+});