Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
fix: preserve cached inspection cues and SDK compatibility
alexshpunt-clanker[bot]9f89ee9db37a
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex f0983d2..a1674f0 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -801,7 +801,7 @@ export class ComputerResources {if (view.kind === "element") {if (!snapshot.target.window_ref) throw new ComputerRequestError("Display observations have no collected UI.");if (view.locator) throw new ComputerRequestError("A Playwright locator crops a browser page; on a native observation, crop by a UI hint of its Read.");- if (view.operation === "text") return session.elementText(snapshot.target.window_ref, address.ref, view.hint);+ if (view.operation === "text") return session.elementText(snapshot.target.window_ref, address.ref, view.hint, signal);if (view.operation === "crop") this.retire(address.ref);return session.observe(snapshot.target.window_ref, signal, undefined, undefined,{ operation: view.operation, observation_id: address.ref, hint: view.hint });@@ -821,7 +821,7 @@ export class ComputerResources {if (view.capture.granularity === "ui" && !snapshot.result.details.ui) throw new ComputerRequestError("This observation has no collected UI. Read a new UI/full window overview.");if (view.capture.granularity === "image" && !snapshot.result.content.some(block => block.type === "image"))throw new ComputerRequestError("This observation has no pixels. Read a new image/full window overview.");- return snapshot.result;+ return snapshot.target.window_ref ? session.presentObservation(snapshot.target.window_ref, snapshot.result, signal) : snapshot.result;}/** Search permitted discovery metadata or the current collected UI without another native walk; the receipt names its machine. */diff --git a/src/provider.ts b/src/provider.tsindex 1c85f2e..6685be3 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -519,11 +519,14 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cwmatches: input => owns(record(input).path),execute: (context, input) => refuseComputerEdit(resources, tool, context.sourceFor("path"), record(input)),});- api.tool("delete").addSemanticHandler({+ const deleteTool = api.tool("delete");+ deleteTool.addSemanticHandler({matches: input => owns(record(input).path),execute: (context, input) => deleteComputer(resources, context.sourceFor("path"), record(input), context.cwd, context.signal),});- api.tool("delete").describeField("path", COMPUTER_DELETE_HELP);+ // Field descriptions are additive on newer IDE SDKs; semantic ownership works on both.+ if ("describeField" in deleteTool && typeof deleteTool.describeField === "function")+ deleteTool.describeField("path", COMPUTER_DELETE_HELP);} })),]);return { resources, guide, attached: () => accepted.size === 3 };diff --git a/src/session.ts b/src/session.tsindex f1fd73a..3484641 100644--- a/src/session.ts+++ b/src/session.ts@@ -1555,9 +1555,10 @@ export class ComputerSession {* The complete text of one element in the current native observation, without another walk. Texts longer* than the walk's 1024-character cut come from the same capture, up to 200,000 characters.*/- elementText(ref: string, id: string, hint: string): Promise<ComputerResult> {+ elementText(ref: string, id: string, hint: string, signal?: AbortSignal): Promise<ComputerResult> {return this.queue(async () => {- const entry = this.entries.get(ref);+ if (signal?.aborted) return this.result("refused", "Element text read cancelled.");+ const entry = await this.current(ref, signal);if (!entry) return this.result("refused", "Unknown or closed window. Discover again.");if (!this.allowed(entry)) return this.denied(entry.ref);const observed = entry.observation;@@ -1568,8 +1569,8 @@ export class ComputerSession {const text = observed.texts?.get(hint) ?? [element.name, element.value].filter(item => item).join("\n");const cut = text.length >= ELEMENT_TEXT_LIMIT;const region = observed.frames.get(element.element_token);- await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) });- if (!this.allowed(entry)) return this.denied(ref);+ await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) }, signal);+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Element text access revoked or cancelled.");return this.result("observed", `Element ${hint} text from this observation: ${text.length} characters${cut ? ", cut at the 200,000-character limit" : ""}. Page it with offset/limit.`,undefined, { ...this.addressed(entry), observation_id: id, element_text: { hint, role: element.role, name: element.name, text, cut } });});@@ -1938,6 +1939,10 @@ export class ComputerSession {undefined, { ...this.addressed(entry), observation_id: observed.id, ui, source: observed.source,agent_image: { ...metadata, route: "cua_snapshot_ui_hints", annotation: rendered.annotation, image_id: observed.id } }, [rendered.image]);}+ const selected = request.hint && ui.elements.find(element => element.hint === request.hint);+ const region = selected && observed.frames.get(selected.element_token);+ await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) }, signal);+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "UI inspection revoked or cancelled; data withheld.");return this.result("observed", `Read-only UI ${request.operation}: ${ui.elements.length}/${ui.matches} matches in the collected snapshot. ${ui.complete === true ? "Native collection reported complete." : "Collection may be partial; absence does not prove nonexistence."}`,undefined, { ...this.addressed(entry), observation_id: observed.id, ui });}diff --git a/test/window-observer.test.ts b/test/window-observer.test.tsindex 5b96329..0f1daa1 100644--- a/test/window-observer.test.ts+++ b/test/window-observer.test.ts@@ -53,6 +53,26 @@ test("change reads present only images in the final returned result", async () =expect(observer.mock.calls[0][0]).toMatchObject({ image: images.at(-1) });expect(observer.mock.calls[0][0]).not.toHaveProperty("image.data", changed.data);});+test("cached image reads and UI inspection announce their existing source without recapturing", async () => {+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = await fixture(observer);+ const ref = await outside(f.session); const resources = new ComputerResources(async () => f.session, async () => f.session.close());+ const overview = (await resources.read(requireComputerAddress(`computer:window/${ref}`), process.cwd()))!;+ observer.mockClear(); const before = vi.mocked(f.driver.call).mock.calls.length;+ const snapshot = requireComputerAddress(`computer:observation/${overview.details.observation_id}`);+ const reread = (await resources.read(snapshot, process.cwd()))!;+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ image: reread.content.find(block => block.type === "image") });+ observer.mockClear();+ const inspected = await resources.read(snapshot, process.cwd(), [`element:inspect,${overview.details.ui!.elements[1].hint}`]);+ expect(inspected!.details.status).toBe("observed");+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "inspect", region: { x: 230, y: 240, width: 120, height: 60 } } });+ expect(observer.mock.calls[0][0]).not.toHaveProperty("image");+ expect(vi.mocked(f.driver.call).mock.calls).toHaveLength(before);+ observer.mockClear(); const abort = new AbortController(); abort.abort();+ const cancelled = await resources.read(snapshot, process.cwd(), [`element:text,${overview.details.ui!.elements[1].hint}`], abort.signal);+ expect(cancelled!.details.status).toBe("refused"); expect(observer).not.toHaveBeenCalled();+});test("overview, requested window zoom and primary-display crop flash their own physical scopes", async () => {const observer = vi.fn<WindowObserver>(async () => ({ status: "shown" })); const f = await fixture(observer);f.setMode("safe"); const opened = await f.session.open("Calculator");