Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

fix: point input captures the window again when its observation was spent by a refused detail or the registry limit (LPT-941)

alexshpunt-clanker[bot]aa0ed3ab01d5
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 22a477c..f6925b1 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -912,6 +912,9 @@ export class ComputerResources {
if (foreground && !isSemanticAction(action)) command.delivery_mode = "foreground";
let target = plan.target;
let view = session.inputView(window);
+ // The registry decides whether an observation still grounds input: a detail Read, a sequence or the registry's
+ // size limit can spend it while the session still names it as the window's current one.
+ if (view.native && !this.snapshots.has(view.native.id)) view = { ...view, native: undefined };
// Actions that only exist on locators take a ref of the latest Read as the locator that names its element.
if (refKind(target) === "hint" && (LOCATOR_ONLY.includes(action) || command.option !== undefined)) target = suggestLocator(this.hinted(view, target as string));
if (command.option !== undefined) {
diff --git a/test/computer-resources.test.ts b/test/computer-resources.test.ts
index ae9c567..ba3313f 100644
--- a/test/computer-resources.test.ts
+++ b/test/computer-resources.test.ts
@@ -2,7 +2,7 @@ import { expect, test, vi } from "vitest";
import sharp from "sharp";
import { computerSource } from "../src/computer-address.ts";
import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";
-import { typePlan } from "../src/computer-tools.ts";
+import { clickPlan, typePlan } from "../src/computer-tools.ts";
import { ComputerSession, type ComputerMode, type Content, type Driver, type DriverResult } from "../src/session.ts";
const cwd = "/synthetic-computer-fixture";
@@ -192,6 +192,34 @@ test("a successful rectangle Read is readonly detail and consumes its overview w
expect(f.inputs()).toHaveLength(0);
});
+// LPT-941: a detail Read spends the window's current observation, also when the region refuses; point input then
+// captures the window again.
+test.each(["region:80,60,40,20", "region:480,300,100,100"])("a point click after a detail Read captures the window again, while a ref of the spent Read still refuses (%s)", async region => {
+ const f = await fixture(), view = await f.read(["image"]);
+ await f.resources.read(view.observation, cwd, [region]).catch(() => undefined);
+ const calls = f.native().length;
+ const clicked = await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, x: 100, y: 70 }));
+ expect(clicked.details.status, JSON.stringify(clicked.content)).toBe("delivered");
+ // The click captured the window first; it did not act on the spent observation.
+ expect(f.native().slice(calls).map(([name]) => name).filter(name => name !== "list_windows").slice(0, 2)).toEqual(["get_window_state", "click"]);
+ const full = await f.read();
+ await f.resources.read(full.observation, cwd, [region]).catch(() => undefined);
+ const inputs = f.inputs().length;
+ await expect(f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, target: full.hint! }))).rejects.toThrow(/is not a ref of this window's latest Read/);
+ expect(f.inputs()).toHaveLength(inputs);
+});
+
+test("a point click after a window sequence captures the window again", async () => {
+ const f = await fixture();
+ await f.read(["image"]);
+ const recorded = await f.resources.sequence(f.window, cwd, ['sequence:{"duration_ms":100,"fps":10,"max_frames":1}']);
+ expect(recorded?.details.status).toBe("observed");
+ const calls = f.native().length;
+ const clicked = await f.resources.input(f.window, cwd, clickPlan({ window: f.window.source, x: 100, y: 70 }));
+ expect(clicked.details.status, JSON.stringify(clicked.content)).toBe("delivered");
+ expect(f.native().slice(calls).map(([name]) => name).filter(name => name !== "list_windows").slice(0, 2)).toEqual(["get_window_state", "click"]);
+});
+
test("invalid JSON and identity overrides are rejected before native input and cannot broaden the bound target", async () => {
const f = await fixture(), view = await f.read(["ui"]);
for (const value of ["not JSON", "[]", JSON.stringify({ action: "click", hint: view.hint, window_ref: "other-window" }),