Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Tests for lean computer-use text (LPT-995)

alexshpunt-clanker[bot]c467e2a33ac5
diff --git a/test/extension-modes.integration.test.ts b/test/extension-modes.integration.test.ts
index 004e1b2..3de596f 100644
--- a/test/extension-modes.integration.test.ts
+++ b/test/extension-modes.integration.test.ts
@@ -68,8 +68,13 @@ test("without Pi Agent IDE, tool_search finds the deferred computer tools and th
expect(getToolResultText(result, "display")).not.toContain("unknown");
expect(getToolResultText(result, "display")).toContain("Display addresses name their machine");
expect(getToolResultText(result, "delete")).toContain("Computer closed");
- expect(getToolResultText(result, "guide")).toMatch(/^# Computer resources/);
+ expect(getToolResultText(result, "guide")).toMatch(/^# Computer\n/);
+ // The guide names the standalone tools, never a placeholder.
+ expect(getToolResultText(result, "guide")).toContain("computer_read");
+ expect(getToolResultText(result, "guide")).not.toMatch(/\{(read|search|delete)\}/);
expect(getToolResultText(result, "session")).toContain("computer:machine/local");
+ // The guide was read already, so the first computer address does not bring it again.
+ expect(getToolResultText(result, "session")).not.toContain("# Guide: computer");
expect(getToolResultText(result, "close")).toContain("Computer closed");
}, 120_000);
@@ -77,10 +82,30 @@ test("an IDE that refuses this plugin API version leaves computer use standalone
const result = await run("incompatible-ide", ["builtin:tool-search", path.join(root, "test/fixtures/incompatible-ide-extension.ts"), computerUse], ["read"], [
{ id: "find", name: "tool_search", arguments: { query: "computer read windows displays" } },
{ id: "session", name: "computer_read", arguments: { path: "computer:session" } },
+ { id: "again", name: "computer_read", arguments: { path: "computer:session" } },
]);
- expect(getToolCallNames(result)).toEqual(["tool_search", "computer_read"]);
+ expect(getToolCallNames(result)).toEqual(["tool_search", "computer_read", "computer_read"]);
expect(getToolExecution(result, "session").isError, getToolResultText(result, "session")).toBe(false);
expect(getToolResultText(result, "session")).toContain("computer:machine/local");
+ // As with the IDE, the first read of a computer address brings the guide with the standalone names, and only once.
+ expect(getToolResultText(result, "session")).toContain("# Guide: computer");
+ expect(getToolResultText(result, "session")).toContain("computer_read of a viewport only looks");
+ expect(getToolResultText(result, "again")).not.toContain("# Guide: computer");
+}, 120_000);
+
+test("without Pi Agent IDE, a script's first computer_read brings the guide to the script's own result", async () => {
+ const result = await run("standalone-codemode", [path.join(root, "test/fixtures/codemode-extension.ts"), computerUse], ["codemode"], [
+ // The script prints nothing of what it read; the guide still reaches the model once.
+ { id: "script", name: "codemode", arguments: { code: `await tools.computer_read({ path: "computer:session" }); await tools.computer_read({ path: "computer:session" }); text("read twice");` } },
+ { id: "after", name: "codemode", arguments: { code: `text(await tools.computer_read({ path: "computer:session" }));` } },
+ { id: "close", name: "codemode", arguments: { code: `await tools.computer_delete({ path: "computer:session" }); text("closed");` } },
+ ]);
+ expect(getToolExecution(result, "script").isError, getToolResultText(result, "script")).toBe(false);
+ expect(getToolResultText(result, "script")).toContain("read twice");
+ expect(getToolResultText(result, "script").split("# Guide: computer").length).toBe(2);
+ expect(getToolResultText(result, "script")).toContain("computer_read of a viewport only looks");
+ expect(getToolResultText(result, "after")).toContain("computer:machine/local");
+ expect(getToolResultText(result, "after")).not.toContain("# Guide: computer");
}, 120_000);
test("with Pi Agent IDE, computer use extends Read, Search and Delete, refuses Write and Insert, and adds only the typed computer tools", async () => {
@@ -110,7 +135,11 @@ test("with Pi Agent IDE, computer use extends Read, Search and Delete, refuses W
"computer_click", "computer_hover", "computer_drag", "computer_scroll", "computer_type", "computer_press", "computer_select", "computer_timeline"]) expect(loaded).toContain(`- ${tool}:`);
for (const id of ["session", "close", "delete"])
expect(getToolExecution(result, id).isError, getToolResultText(result, id)).toBe(false);
- expect(getToolResultText(result, "session")).toContain("computer:session");
+ expect(getToolResultText(result, "session")).toContain("computer:machine/local");
+ // The first Read of a computer address brings the guide, with the IDE's tool names filled in.
+ expect(getToolResultText(result, "session")).toContain("# Guide: computer");
+ expect(getToolResultText(result, "session")).toContain("read of a viewport only looks");
+ expect(getToolResultText(result, "session")).not.toMatch(/\{(read|search|delete)\}|computer_read/);
// Write and Insert only name the typed tool that does what they were sent; nothing reaches a machine.
for (const id of ["insert", "write"]) expect(getToolExecution(result, id).isError).toBe(true);
expect(getToolResultText(result, "insert")).toContain("Insert does not act on computer resources; nothing was sent. Use computer_open.");
diff --git a/test/fixtures/codemode-extension.ts b/test/fixtures/codemode-extension.ts
new file mode 100644
index 0000000..07155b0
--- /dev/null
+++ b/test/fixtures/codemode-extension.ts
@@ -0,0 +1,6 @@
+import { createCodemodeExtension, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+/** Turn Codemode on in a real Pi run, so a test can call the computer tools from a script. */
+export default function codemode(pi: ExtensionAPI): void {
+ createCodemodeExtension({ mode: "on" })(pi);
+}
diff --git a/test/fixtures/standalone-computer.ts b/test/fixtures/standalone-computer.ts
index 18b9ff7..77f6fb2 100644
--- a/test/fixtures/standalone-computer.ts
+++ b/test/fixtures/standalone-computer.ts
@@ -3,6 +3,7 @@ import { ComputerResources } from "../../src/computer-resources.ts";
import type { ComputerMode, ComputerResult, ComputerSession } from "../../src/session.ts";
import { registerStandaloneTools } from "../../src/standalone-tools.ts";
import { registerComputerTools } from "../../src/computer-tools.ts";
+import { STANDALONE_TOOL_NAMES } from "../../src/tool-names.ts";
export const STANDALONE_TOOLS = ["computer_read", "computer_search", "computer_delete", "computer_open", "computer_window", "computer_display", "computer_wait",
"computer_browser", "computer_files", "computer_clipboard", "computer_record", "computer_click", "computer_hover", "computer_drag", "computer_scroll",
@@ -16,7 +17,7 @@ export function registerFixtureComputerTools(pi: ExtensionAPI, getSession: (cwd:
closeSession: () => Promise<ComputerResult>, getMode?: () => ComputerMode): ComputerResources {
const resources = new ComputerResources(getSession, closeSession, undefined, undefined, getMode);
pi.on("session_start", () => {
- registerComputerTools(pi, resources);
+ registerComputerTools(pi, resources, STANDALONE_TOOL_NAMES);
registerStandaloneTools(pi, resources, "Fixture computer guide.");
pi.setActiveTools([...new Set([...pi.getActiveTools(), ...STANDALONE_TOOLS])]);
});
diff --git a/test/tool-names.test.ts b/test/tool-names.test.ts
new file mode 100644
index 0000000..f2b8808
--- /dev/null
+++ b/test/tool-names.test.ts
@@ -0,0 +1,24 @@
+import { StringEnum, Type, validateToolArguments } from "@earendil-works/pi-ai";
+import { expect, test } from "vitest";
+import { fillToolNames, IDE_TOOL_NAMES, STANDALONE_TOOL_NAMES } from "../src/tool-names.ts";
+
+const schema = Type.Object({
+ window: Type.String({ description: "Window address from {search}, {read} or a name." }),
+ views: Type.Optional(Type.Array(Type.String(), { description: "Views of one {read}; {delete} releases it." })),
+}, { additionalProperties: false });
+
+test("tool names are filled per mode and the declared JSON keeps its shape", () => {
+ for (const names of [IDE_TOOL_NAMES, STANDALONE_TOOL_NAMES]) {
+ const filled = fillToolNames(schema, names);
+ const expected = JSON.stringify(schema).replaceAll("{search}", names.search).replaceAll("{read}", names.read).replaceAll("{delete}", names.delete);
+ expect(JSON.stringify(filled)).toBe(expected);
+ }
+ expect((fillToolNames(schema, STANDALONE_TOOL_NAMES).properties.window as { description?: string }).description).toBe("Window address from computer_search, computer_read or a name.");
+});
+
+test("a filled enum schema still accepts its values and rejects others", () => {
+ const tool = { name: "t", description: "", parameters: fillToolNames(Type.Object({ mode: StringEnum(["wait", "check"] as const, { description: "{read}" }) }), IDE_TOOL_NAMES) };
+ const call = (mode: string) => validateToolArguments(tool, { type: "toolCall", id: "1", name: "t", arguments: { mode } });
+ expect(call("check")).toEqual({ mode: "check" });
+ expect(() => call("bogus")).toThrow();
+});