Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Tests read short addresses from presented text (LPT-932)
alexshpunt-clanker[bot]78f61cbcaa27
diff --git a/test/extension.integration.test.ts b/test/extension.integration.test.tsindex b4a5553..bea7b1f 100644--- a/test/extension.integration.test.ts+++ b/test/extension.integration.test.ts@@ -40,8 +40,8 @@ test("standalone Pi opens two windows with first observations, rejects inventedtarget: { pid: 42, window_id: 101 }, state: { elements: [{ element_token: "s1:0" }] } });expect(second).toMatchObject({ status: "observed", target: { pid: 42, window_id: 102 } });expect(second.window_ref).not.toBe(first.window_ref);- expect(getToolResultText(result, "first")).toContain(`computer:window/${first.window_ref}`);- expect(getToolResultText(result, "first")).toContain(`computer:observation/${first.observation_id}`);+ // The text shows this run's short addresses (LPT-932); the long references stay in the details.+ expect(getToolResultText(result, "first")).toMatch(/^Window: computer:local\/w1[a-z]{2}\nObservation: computer:local\/o1[a-z]{2}$/m);expect(getToolExecution(result, "first").isError).toBe(false);expect(getToolExecution(result, "stale").isError).toBe(true);expect(getToolResultText(result, "stale")).toContain("Observation reference expired");diff --git a/test/fixtures/addresses.ts b/test/fixtures/addresses.tsnew file mode 100644index 0000000..2c88c38--- /dev/null+++ b/test/fixtures/addresses.ts@@ -0,0 +1,7 @@+/** Letters of the short address forms (computer:<machine>/<letter><n><tag>). */+const LETTERS = { window: "w", observation: "o", sequence: "s", "window-state": "t", app: "a", sandbox: "b" } as const;++/** A pattern that finds a computer address of one kind in presented text, in its long or short form. */+export function addressPattern(kind: keyof typeof LETTERS, flags = ""): RegExp {+ return new RegExp(`computer:(?:${kind}/[a-zA-Z0-9-]+|[a-z0-9_-]+/${LETTERS[kind]}[0-9]+[a-z]{2})`, flags);+}diff --git a/test/fixtures/coordinate-scroll-extension.ts b/test/fixtures/coordinate-scroll-extension.tsindex fcaf95d..fc8b02b 100644--- a/test/fixtures/coordinate-scroll-extension.ts+++ b/test/fixtures/coordinate-scroll-extension.ts@@ -1,3 +1,4 @@+import { addressPattern } from "./addresses.ts";import { readFile } from "node:fs/promises";import { Type } from "@earendil-works/pi-ai";import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";@@ -17,7 +18,7 @@ function field(value: unknown, key: string): unknown {return undefined;}const reference = (result: Result, kind: "window" | "observation") => {- const match = textOf(result).match(new RegExp(`computer:${kind}/[a-zA-Z0-9-]+`));+ const match = textOf(result).match(addressPattern(kind));if (!match) throw Error(`No ${kind} resource in actual IDE result: ${textOf(result)}`);return match[0];};diff --git a/test/fixtures/ide-provider-extension.ts b/test/fixtures/ide-provider-extension.tsindex 24967d7..4d6ddc3 100644--- a/test/fixtures/ide-provider-extension.ts+++ b/test/fixtures/ide-provider-extension.ts@@ -1,3 +1,4 @@+import { addressPattern } from "./addresses.ts";import { readFile } from "node:fs/promises";import path from "node:path";import { pathToFileURL } from "node:url";@@ -9,7 +10,7 @@ interface Result { isError?: boolean; content: { type: string; text?: string; daconst textOf = (result: Result) => result.content.filter(block => block.type === "text").map(block => block.text).join("\n");const assert = (condition: unknown, message: string): void => { if (!condition) throw Error(message); };const reference = (result: Result, kind: "app" | "window" | "observation" | "sequence") => {- const match = textOf(result).match(new RegExp(`computer:${kind}/[a-zA-Z0-9-]+`));+ const match = textOf(result).match(addressPattern(kind));if (!match) throw Error(`No ${kind} resource in real tool result: ${textOf(result)}`);return match[0].split("?")[0];};diff --git a/test/fixtures/standalone-computer.ts b/test/fixtures/standalone-computer.tsindex 68d55c3..69ce038 100644--- a/test/fixtures/standalone-computer.ts+++ b/test/fixtures/standalone-computer.ts@@ -26,7 +26,10 @@ export function registerFixtureComputerTools(pi: ExtensionAPI, getSession: (cwd:export const SCRIPT_HELPERS = String.raw`const body = value => typeof value === "string" ? value : value.note ?? value.text ?? "";const address = (value, kind, label) => {- const match = body(value).match(new RegExp((label ? label + ": " : "") + "(computer:" + kind + "/[\\w-]+)"));+ // Results show short forms (computer:local/o3qk); a labelled line gives the address of either form.+ const letter = { window: "w", observation: "o", sequence: "s", sandbox: "b", app: "a" }[kind];+ const match = body(value).match(label ? new RegExp("^" + label + ": (computer:[\\w/-]+)", "m")+ : new RegExp("(computer:" + kind + "/[\\w-]+|computer:[a-z0-9_-]+/" + letter + "[0-9]+[a-z]{2})"));if (!match) throw Error("No " + kind + " address in: " + body(value).slice(0, 600));return match[1];};diff --git a/test/fixtures/window-management-extension.ts b/test/fixtures/window-management-extension.tsindex d330a8c..3af2062 100644--- a/test/fixtures/window-management-extension.ts+++ b/test/fixtures/window-management-extension.ts@@ -1,3 +1,4 @@+import { addressPattern } from "./addresses.ts";import { readFile } from "node:fs/promises";import path from "node:path";import { pathToFileURL } from "node:url";@@ -16,7 +17,7 @@ function find(value: unknown, key: string): unknown {return undefined;}function reference(result: Result, kind: "window" | "window-state" | "observation"): string {- const found = body(result).match(new RegExp(`computer:${kind}/[a-zA-Z0-9-]+`));+ const found = body(result).match(addressPattern(kind));assert(found, `No ${kind} resource: ${body(result)}`); return found[0];}function semantic(result: Result): Record<string, unknown> {diff --git a/test/fixtures/window-restore-extension.ts b/test/fixtures/window-restore-extension.tsindex 002374b..bf10d45 100644--- a/test/fixtures/window-restore-extension.ts+++ b/test/fixtures/window-restore-extension.ts@@ -1,3 +1,4 @@+import { addressPattern } from "./addresses.ts";import { readFile } from "node:fs/promises";import path from "node:path";import { pathToFileURL } from "node:url";@@ -15,7 +16,7 @@ function find(value: unknown, key: string): unknown {return undefined;}function reference(result: Result, kind: "window" | "window-state" | "observation"): string {- const found = body(result).match(new RegExp(`computer:${kind}/[a-zA-Z0-9-]+`));+ const found = body(result).match(addressPattern(kind));assert(found, `No ${kind} resource in native tool receipt: ${body(result)}`);return found[0];}diff --git a/test/pointer-hover.integration.test.ts b/test/pointer-hover.integration.test.tsindex bcb796e..2bbf5aa 100644--- a/test/pointer-hover.integration.test.ts+++ b/test/pointer-hover.integration.test.ts@@ -90,7 +90,7 @@ text("Pointer contract passed: admission, receipts and no replay");`);test("real Pi never sends pointer input through Safe-disallowed, closed or reused window references", async () => {const { calls } = await scenario("native-pointer-access", `const listed=await windows();-const external=listed.split("\\n").find(line=>line.includes("| Unowned fixture |")).match(/computer:window\\/[\\w-]+/)[0];+const external=listed.split("\\n").find(line=>line.includes("| Unowned fixture |")).match(/computer:[\\w-]+\\/w\\d+[a-z]{2}/)[0];const externalView=await observe(external);await control("safe");if(await act(externalView,{action:"move",x:100,y:80,delivery_mode:"foreground"})!=="refused")throw Error("Safe sent input to external window");diff --git a/test/selected-displays.integration.test.ts b/test/selected-displays.integration.test.tsindex c4f2a03..627d6b5 100644--- a/test/selected-displays.integration.test.ts+++ b/test/selected-displays.integration.test.ts@@ -13,7 +13,7 @@ for (const mode of ["yolo", "safe"] as const) {await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath,args: [path.resolve("test/fixtures/selected-display-driver.mjs"), log] }));const discover = `const r=await tools.computer_read({path:"computer:machine/local/displays"});-const rows=r.split("\\n").filter(line=>line.startsWith("computer:machine/local/display/"));+const rows=r.split("\\n").filter(line=>/^computer:local\\/d\\d+[a-z]{2} /.test(line));if(rows.length!==2)throw Error("Native discovery failed");const row=rows.find(line=>JSON.parse(line.slice(line.lastIndexOf(" | ")+3)).bounds.x<0);store("monitor",row.split(" | ")[0]);text(row);`;diff --git a/test/shell-desktop.integration.test.ts b/test/shell-desktop.integration.test.tsindex 38f22b1..10b6253 100644--- a/test/shell-desktop.integration.test.ts+++ b/test/shell-desktop.integration.test.ts@@ -16,7 +16,7 @@ for (const mode of ["yolo", "safe"] as const) {args: [path.resolve("test/fixtures/shell-desktop-driver.mjs"), log] }));const discover = `${SCRIPT_HELPERS}const listed=await windows();-const refs=[...listed.matchAll(/computer:window\\/[\\w-]+/g)].map(match=>match[0]);+const refs=[...listed.matchAll(/computer:[\\w-]+\\/w\\d+[a-z]{2}/g)].map(match=>match[0]);if(refs.length!==2)throw Error("Both windows missing");store("ref",refs[0]);text(listed);`;const observe = `const r=await tools.computer_read({path:load("ref"),views:['capture:{"hints":false}']});diff --git a/test/ui-resources.integration.test.ts b/test/ui-resources.integration.test.tsindex 9605dd2..5042f12 100644--- a/test/ui-resources.integration.test.ts+++ b/test/ui-resources.integration.test.ts@@ -15,7 +15,7 @@ test("real Pi Codemode refines full resources locally and dispatches only a curr// Any refused input spends its observation, so each refusal check acts on a fresh overview.const code = `${SCRIPT_HELPERS}const listed = await windows();-const [win, otherWin] = [...listed.matchAll(/computer:window\\/[\\w-]+/g)].map(match => match[0]);+const [win, otherWin] = [...listed.matchAll(/computer:[\\w-]+\\/w\\d+[a-z]{2}/g)].map(match => match[0]);const read = views => tools.computer_read({path:win,...(views?{views}:{})});// Only local refinement is under test here: clicks skip the default successor, so every native capture is a read.const click = async (view,hint) => (await insert(observation(view),{action:"click",hint,successor:false})).outcome;