Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

feat: a native Read and a DOM Read of one window keep each other's observations (LPT-927 part c)

alexshpunt-clanker[bot]5b128cd644b2
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 3f554ba..f340006 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -100,6 +100,8 @@ const SETTLE_MS = 300;
const STEADY_MS = 1_000;
/** What an observation shows: a window, or one machine's display (display references are per machine; each has a primary). */
type ObservationTarget = { window_ref: string; display?: never; machine?: never } | { display: string; machine: string; window_ref?: never };
+/** Whether an observation is a page's DOM snapshot or a native capture of a window or display. */
+const observationKind = (result: ComputerResult) => result.details.dom ? "dom" : "native";
const sameTarget = (a: ObservationTarget, b: ObservationTarget) => a.window_ref === b.window_ref && a.display === b.display && a.machine === b.machine;
/** Where a request runs: the session that issued its reference, with that session's machine and sandbox. */
interface Owner { session: ComputerSession; machine: string; sandbox?: string }
@@ -550,16 +552,18 @@ export class ComputerResources {
if (!entry) throw new ComputerRequestError("Observation reference expired. Read a new window/display overview.");
return entry;
}
- private retireWindow(ref: string): void {
+ /** Forget a window's observations: all of them, or only its DOM snapshots or only its native captures, which a Read of the other kind keeps. */
+ private retireWindow(ref: string, kind?: "dom" | "native"): void {
for (const [id, entry] of this.windowStates) if (entry.window_ref === ref) this.windowStates.delete(id);
- for (const [id, entry] of this.snapshots) if (entry.target.window_ref === ref) this.snapshots.delete(id);
+ for (const [id, entry] of this.snapshots) if (entry.target.window_ref === ref && (!kind || observationKind(entry.result) === kind)) this.snapshots.delete(id);
}
private remember(target: ObservationTarget, result: ComputerResult): ComputerResult {
const id = result.details.observation_id ?? result.details.agent_image?.image_id;
// A delivered DOM action carries the page's next snapshot, which becomes the current observation.
const observed = result.details.status === "observed" || result.details.status === "delivered" && result.details.dom !== undefined;
if (!observed || typeof id !== "string") return result;
- for (const [old, entry] of this.snapshots) if (sameTarget(entry.target, target)) this.snapshots.delete(old);
+ // A window keeps one DOM snapshot and one native capture: a new one replaces only the earlier one of its kind.
+ for (const [old, entry] of this.snapshots) if (sameTarget(entry.target, target) && observationKind(entry.result) === observationKind(result)) this.snapshots.delete(old);
this.snapshots.set(id, { target, result });
if (target.window_ref !== undefined && result.details.ui) {
this.lastUi.delete(target.window_ref);
@@ -733,8 +737,8 @@ export class ComputerResources {
}
if (view.kind === "frames") throw new ComputerRequestError("Frame selection needs a sequence address.");
if (address.kind === "window") {
- // Fresh window Read retires both provider handles before any native acquisition.
- this.retireWindow(address.ref);
+ // A fresh window Read retires the earlier observation of its kind; a DOM snapshot and a native capture coexist.
+ this.retireWindow(address.ref, view.kind === "dom" ? "dom" : view.kind === "state" ? undefined : "native");
if (view.kind === "state") {
const result = await session.observeWindowState(address.ref, signal);
if (result.details.status === "observed" && result.details.window_state_id) {
@@ -766,7 +770,7 @@ export class ComputerResources {
return session.webImage(ref, address.ref, view.operation, { ref: view.hint }, signal);
}
if (view.kind === "dom" && Object.keys(view.options).length) {
- this.retireWindow(ref);
+ this.retireWindow(ref, "dom");
return this.remember({ window_ref: ref }, await session.observeDom(ref, view.options, signal));
}
if (view.kind === "overview" && view.capture.granularity === "image" && Object.keys(view.capture).length === 1)
diff --git a/src/session.ts b/src/session.ts
index fb91dd9..1d1b142 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -975,7 +975,8 @@ export class ComputerSession {
}
private async capture(entry: WindowEntry, signal?: AbortSignal, query?: string, options: UiCaptureOptions = {}, quiet = false): Promise<ComputerResult> {
validateUiCapture(options);
- entry.dom = undefined; entry.windowState = undefined;
+ // A native capture keeps the page's DOM snapshot: reading changes nothing, so both stay current until input.
+ entry.windowState = undefined;
const granularity = options.granularity ?? "full";
const hasUi = granularity !== "image"; const hasImage = granularity !== "ui";
if (!hasUi && query !== undefined) return this.result("refused", "Image-only observation cannot query UI labels.");
@@ -1111,7 +1112,8 @@ export class ComputerSession {
return this.queue(async () => {
const entry = await this.webEntry(ref, signal);
if (!("ref" in entry)) return entry;
- entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
+ // A DOM snapshot keeps the window's native capture: reading changes nothing, so both stay current until input.
+ entry.dom = undefined; entry.windowState = undefined;
try {
const observed = this.webObservation(entry, await entry.web!.page("snapshot", options, signal));
return this.result("observed", "Fresh Playwright aria snapshot. Use [ref=...] values as target.ref while this stays the latest snapshot of the window.",
@@ -1704,7 +1706,8 @@ export class ComputerSession {
if (signal?.aborted) return finish(this.result("refused", "Cancelled during action mark; input was not dispatched."), "Cancelled during action mark.");
if (!this.allowed(entry)) return finish(this.denied(entry.ref), "Permission revoked during action mark.");
if (batch && performance.now() - started >= SEQUENCE_ELAPSED_MS) return finish(this.result("refused", "Sequence time budget reached."), `${SEQUENCE_ELAPSED_MS / 60000} min elapsed budget reached during admission; no more steps dispatched.`);
- entry.observation = undefined; entry.detail = undefined;
+ // Input may change the page, so the window's DOM snapshot retires with its native capture.
+ entry.observation = undefined; entry.detail = undefined; entry.dom = undefined;
const pointerAction = isPointerMove(step.action) || isCoordinateScroll(step) || ["click", "right_click", "double_click", "drag"].includes(step.action) || (step.action === "hold" && !!step.actions?.length);
if (pointerAction) entry.lastAction = { reason: "Pointer outcome is not yet confirmed." };
const requested = { ...step, ...(step.text !== undefined ? { text_length: step.text.length } : {}),
diff --git a/test/live/typed-input.live.ts b/test/live/typed-input.live.ts
index e638f66..96e6466 100644
--- a/test/live/typed-input.live.ts
+++ b/test/live/typed-input.live.ts
@@ -66,6 +66,27 @@ test("one action vocabulary drives a native window by locator, ref and point", a
} finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
});
+test("points of a native canvas draw strokes that the receipt shows as changed pixels", async () => {
+ const opened = await live.insert(live.machine, { action: "open", app: "Paint" });
+ expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
+ const window = `computer:window/${opened.details.window_ref}`;
+ try {
+ const image = await live.read(window, ["image"]);
+ const size = image.details.state as { screenshot_width: number; screenshot_height: number };
+ const middle = { x: Math.round(size.screenshot_width / 2), y: Math.round(size.screenshot_height * 0.6) };
+ const stroke = await act(window, dragPlan({ window, from: { x: middle.x - 120, y: middle.y }, to: { x: middle.x + 120, y: middle.y + 40 }, via: [{ x: middle.x, y: middle.y - 60 }] }));
+ expect(stroke.result.details.status, stroke.body).toBe("delivered");
+ expect(stroke.body).toMatch(/Pixels changed in \d+ area/);
+ const line = await act(window, timelinePlan({ window, events: [{ at_ms: 0, type: "move", x: middle.x - 100, y: middle.y + 80 }, { at_ms: 20, type: "button_down" },
+ { at_ms: 60, type: "move", x: middle.x, y: middle.y + 100 }, { at_ms: 100, type: "move", x: middle.x + 100, y: middle.y + 80 }, { at_ms: 140, type: "button_up" }] }));
+ expect(line.result.details.status, line.body).toBe("delivered");
+ expect(line.body).toMatch(/Pixels changed in \d+ area/);
+ } finally {
+ // Paint asks about the unsaved drawing, so the window the session opened is ended without saving.
+ await deleteComputer(live.resources, `${window}?force=true`, { path: `${window}?force=true` }, live.cwd);
+ }
+});
+
test("points of a browser page take clicks, wheel, drags, held keys and a pressure pen through Playwright", async () => {
const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/input` });
expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
@@ -100,8 +121,14 @@ test("the same vocabulary drives a browser page through Playwright by locator an
const window = `computer:window/${opened.details.window_ref}`;
try {
expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Ada" }))).result.details.status).toBe("delivered");
+ // A native Read and a DOM Read of one window keep each other: the browser's own Reload button stays a usable ref.
+ const reload = /button "Reload" \[ref=([0-9a-z]+:[a-z]+)\]/.exec(presentComputerResult(await live.read(window, ["aria"])))![1];
+ expect((await live.read(window, ["dom"])).details.status).toBe("observed");
+ const reloaded = await act(window, clickPlan({ window, target: reload }));
+ expect(reloaded.result.details.status, reloaded.body).toBe("delivered");
+ expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Ada", replace: true }))).result.details.status).toBe("delivered");
const dom = presentComputerResult(await live.read(window, ["dom"]));
- const save = /button "Save" \[ref=(e\d+)\]/.exec(dom)![1];
+ const save = /button "Save" \[ref=((?:f\d+)?e\d+)\]/.exec(dom)![1];
const clicked = await act(window, clickPlan({ window, target: save }));
expect(clicked.result.details.status, clicked.body).toBe("delivered");
expect(await holds(window, [{ target: { text: "saved Ada" }, is: "visible" }])).toBe("satisfied");