Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
fix: point input captures the window again when its observation was spent by a refused detail or the registry limit (LPT-941)
alexshpunt-clanker[bot]aa0ed3ab01d5
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 22a477c..f6925b1 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -912,6 +912,9 @@ export class ComputerResources {if (foreground && !isSemanticAction(action)) command.delivery_mode = "foreground";let target = plan.target;let view = session.inputView(window);+ // The registry decides whether an observation still grounds input: a detail Read, a sequence or the registry's+ // size limit can spend it while the session still names it as the window's current one.+ if (view.native && !this.snapshots.has(view.native.id)) view = { ...view, native: undefined };// Actions that only exist on locators take a ref of the latest Read as the locator that names its element.if (refKind(target) === "hint" && (LOCATOR_ONLY.includes(action) || command.option !== undefined)) target = suggestLocator(this.hinted(view, target as string));if (command.option !== undefined) {diff --git a/test/computer-resources.test.ts b/test/computer-resources.test.tsindex ae9c567..ba3313f 100644--- a/test/computer-resources.test.ts+++ b/test/computer-resources.test.ts@@ -2,7 +2,7 @@ import { expect, test, vi } from "vitest";import sharp from "sharp";import { computerSource } from "../src/computer-address.ts";import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";-import { typePlan } from "../src/computer-tools.ts";+import { clickPlan, typePlan } from "../src/computer-tools.ts";import { ComputerSession, type ComputerMode, type Content, type Driver, type DriverResult } from "../src/session.ts";const cwd = "/synthetic-computer-fixture";@@ -192,6 +192,34 @@ test("a successful rectangle Read is readonly detail and consumes its overview wexpect(f.inputs()).toHaveLength(0);});+// LPT-941: a detail Read spends the window's current observation, also when the region refuses; point input then+// captures the window again.+test.each(["region:80,60,40,20", "region:480,300,100,100"])("a point click after a detail Read captures the window again, while a ref of the spent Read still refuses (%s)", async region => {+ const f = await fixture(), view = await f.read(["image"]);+ await f.resources.read(view.observation, cwd, [region]).catch(() => undefined);+ const calls = f.native().length;+ const clicked = await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, x: 100, y: 70 }));+ expect(clicked.details.status, JSON.stringify(clicked.content)).toBe("delivered");+ // The click captured the window first; it did not act on the spent observation.+ expect(f.native().slice(calls).map(([name]) => name).filter(name => name !== "list_windows").slice(0, 2)).toEqual(["get_window_state", "click"]);+ const full = await f.read();+ await f.resources.read(full.observation, cwd, [region]).catch(() => undefined);+ const inputs = f.inputs().length;+ await expect(f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, target: full.hint! }))).rejects.toThrow(/is not a ref of this window's latest Read/);+ expect(f.inputs()).toHaveLength(inputs);+});++test("a point click after a window sequence captures the window again", async () => {+ const f = await fixture();+ await f.read(["image"]);+ const recorded = await f.resources.sequence(f.window, cwd, ['sequence:{"duration_ms":100,"fps":10,"max_frames":1}']);+ expect(recorded?.details.status).toBe("observed");+ const calls = f.native().length;+ const clicked = await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, x: 100, y: 70 }));+ expect(clicked.details.status, JSON.stringify(clicked.content)).toBe("delivered");+ expect(f.native().slice(calls).map(([name]) => name).filter(name => name !== "list_windows").slice(0, 2)).toEqual(["get_window_state", "click"]);+});+test("invalid JSON and identity overrides are rejected before native input and cannot broaden the bound target", async () => {const f = await fixture(), view = await f.read(["ui"]);for (const value of ["not JSON", "[]", JSON.stringify({ action: "click", hint: view.hint, window_ref: "other-window" }),