Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

fix: preserve cached inspection cues and SDK compatibility

alexshpunt-clanker[bot]9f89ee9db37a
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index f0983d2..a1674f0 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -801,7 +801,7 @@ export class ComputerResources {
if (view.kind === "element") {
if (!snapshot.target.window_ref) throw new ComputerRequestError("Display observations have no collected UI.");
if (view.locator) throw new ComputerRequestError("A Playwright locator crops a browser page; on a native observation, crop by a UI hint of its Read.");
- if (view.operation === "text") return session.elementText(snapshot.target.window_ref, address.ref, view.hint);
+ if (view.operation === "text") return session.elementText(snapshot.target.window_ref, address.ref, view.hint, signal);
if (view.operation === "crop") this.retire(address.ref);
return session.observe(snapshot.target.window_ref, signal, undefined, undefined,
{ operation: view.operation, observation_id: address.ref, hint: view.hint });
@@ -821,7 +821,7 @@ export class ComputerResources {
if (view.capture.granularity === "ui" && !snapshot.result.details.ui) throw new ComputerRequestError("This observation has no collected UI. Read a new UI/full window overview.");
if (view.capture.granularity === "image" && !snapshot.result.content.some(block => block.type === "image"))
throw new ComputerRequestError("This observation has no pixels. Read a new image/full window overview.");
- return snapshot.result;
+ return snapshot.target.window_ref ? session.presentObservation(snapshot.target.window_ref, snapshot.result, signal) : snapshot.result;
}
/** Search permitted discovery metadata or the current collected UI without another native walk; the receipt names its machine. */
diff --git a/src/provider.ts b/src/provider.ts
index 1c85f2e..6685be3 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -519,11 +519,14 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw
matches: input => owns(record(input).path),
execute: (context, input) => refuseComputerEdit(resources, tool, context.sourceFor("path"), record(input)),
});
- api.tool("delete").addSemanticHandler({
+ const deleteTool = api.tool("delete");
+ deleteTool.addSemanticHandler({
matches: input => owns(record(input).path),
execute: (context, input) => deleteComputer(resources, context.sourceFor("path"), record(input), context.cwd, context.signal),
});
- api.tool("delete").describeField("path", COMPUTER_DELETE_HELP);
+ // Field descriptions are additive on newer IDE SDKs; semantic ownership works on both.
+ if ("describeField" in deleteTool && typeof deleteTool.describeField === "function")
+ deleteTool.describeField("path", COMPUTER_DELETE_HELP);
} })),
]);
return { resources, guide, attached: () => accepted.size === 3 };
diff --git a/src/session.ts b/src/session.ts
index f1fd73a..3484641 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -1555,9 +1555,10 @@ export class ComputerSession {
* The complete text of one element in the current native observation, without another walk. Texts longer
* than the walk's 1024-character cut come from the same capture, up to 200,000 characters.
*/
- elementText(ref: string, id: string, hint: string): Promise<ComputerResult> {
+ elementText(ref: string, id: string, hint: string, signal?: AbortSignal): Promise<ComputerResult> {
return this.queue(async () => {
- const entry = this.entries.get(ref);
+ if (signal?.aborted) return this.result("refused", "Element text read cancelled.");
+ const entry = await this.current(ref, signal);
if (!entry) return this.result("refused", "Unknown or closed window. Discover again.");
if (!this.allowed(entry)) return this.denied(entry.ref);
const observed = entry.observation;
@@ -1568,8 +1569,8 @@ export class ComputerSession {
const text = observed.texts?.get(hint) ?? [element.name, element.value].filter(item => item).join("\n");
const cut = text.length >= ELEMENT_TEXT_LIMIT;
const region = observed.frames.get(element.element_token);
- await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) });
- if (!this.allowed(entry)) return this.denied(ref);
+ await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) }, signal);
+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Element text access revoked or cancelled.");
return this.result("observed", `Element ${hint} text from this observation: ${text.length} characters${cut ? ", cut at the 200,000-character limit" : ""}. Page it with offset/limit.`,
undefined, { ...this.addressed(entry), observation_id: id, element_text: { hint, role: element.role, name: element.name, text, cut } });
});
@@ -1938,6 +1939,10 @@ export class ComputerSession {
undefined, { ...this.addressed(entry), observation_id: observed.id, ui, source: observed.source,
agent_image: { ...metadata, route: "cua_snapshot_ui_hints", annotation: rendered.annotation, image_id: observed.id } }, [rendered.image]);
}
+ const selected = request.hint && ui.elements.find(element => element.hint === request.hint);
+ const region = selected && observed.frames.get(selected.element_token);
+ await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) }, signal);
+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "UI inspection revoked or cancelled; data withheld.");
return this.result("observed", `Read-only UI ${request.operation}: ${ui.elements.length}/${ui.matches} matches in the collected snapshot. ${ui.complete === true ? "Native collection reported complete." : "Collection may be partial; absence does not prove nonexistence."}`,
undefined, { ...this.addressed(entry), observation_id: observed.id, ui });
}
diff --git a/test/window-observer.test.ts b/test/window-observer.test.ts
index 5b96329..0f1daa1 100644
--- a/test/window-observer.test.ts
+++ b/test/window-observer.test.ts
@@ -53,6 +53,26 @@ test("change reads present only images in the final returned result", async () =
expect(observer.mock.calls[0][0]).toMatchObject({ image: images.at(-1) });
expect(observer.mock.calls[0][0]).not.toHaveProperty("image.data", changed.data);
});
+test("cached image reads and UI inspection announce their existing source without recapturing", async () => {
+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = await fixture(observer);
+ const ref = await outside(f.session); const resources = new ComputerResources(async () => f.session, async () => f.session.close());
+ const overview = (await resources.read(requireComputerAddress(`computer:window/${ref}`), process.cwd()))!;
+ observer.mockClear(); const before = vi.mocked(f.driver.call).mock.calls.length;
+ const snapshot = requireComputerAddress(`computer:observation/${overview.details.observation_id}`);
+ const reread = (await resources.read(snapshot, process.cwd()))!;
+ expect(observer).toHaveBeenCalledTimes(1);
+ expect(observer.mock.calls[0][0]).toMatchObject({ image: reread.content.find(block => block.type === "image") });
+ observer.mockClear();
+ const inspected = await resources.read(snapshot, process.cwd(), [`element:inspect,${overview.details.ui!.elements[1].hint}`]);
+ expect(inspected!.details.status).toBe("observed");
+ expect(observer).toHaveBeenCalledTimes(1);
+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "inspect", region: { x: 230, y: 240, width: 120, height: 60 } } });
+ expect(observer.mock.calls[0][0]).not.toHaveProperty("image");
+ expect(vi.mocked(f.driver.call).mock.calls).toHaveLength(before);
+ observer.mockClear(); const abort = new AbortController(); abort.abort();
+ const cancelled = await resources.read(snapshot, process.cwd(), [`element:text,${overview.details.ui!.elements[1].hint}`], abort.signal);
+ expect(cancelled!.details.status).toBe("refused"); expect(observer).not.toHaveBeenCalled();
+});
test("overview, requested window zoom and primary-display crop flash their own physical scopes", async () => {
const observer = vi.fn<WindowObserver>(async () => ({ status: "shown" })); const f = await fixture(observer);
f.setMode("safe"); const opened = await f.session.open("Calculator");