Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
feat: a native Read and a DOM Read of one window keep each other's observations (LPT-927 part c)
alexshpunt-clanker[bot]5b128cd644b2
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 3f554ba..f340006 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -100,6 +100,8 @@ const SETTLE_MS = 300;const STEADY_MS = 1_000;/** What an observation shows: a window, or one machine's display (display references are per machine; each has a primary). */type ObservationTarget = { window_ref: string; display?: never; machine?: never } | { display: string; machine: string; window_ref?: never };+/** Whether an observation is a page's DOM snapshot or a native capture of a window or display. */+const observationKind = (result: ComputerResult) => result.details.dom ? "dom" : "native";const sameTarget = (a: ObservationTarget, b: ObservationTarget) => a.window_ref === b.window_ref && a.display === b.display && a.machine === b.machine;/** Where a request runs: the session that issued its reference, with that session's machine and sandbox. */interface Owner { session: ComputerSession; machine: string; sandbox?: string }@@ -550,16 +552,18 @@ export class ComputerResources {if (!entry) throw new ComputerRequestError("Observation reference expired. Read a new window/display overview.");return entry;}- private retireWindow(ref: string): void {+ /** Forget a window's observations: all of them, or only its DOM snapshots or only its native captures, which a Read of the other kind keeps. */+ private retireWindow(ref: string, kind?: "dom" | "native"): void {for (const [id, entry] of this.windowStates) if (entry.window_ref === ref) this.windowStates.delete(id);- for (const [id, entry] of this.snapshots) if (entry.target.window_ref === ref) this.snapshots.delete(id);+ for (const [id, entry] of this.snapshots) if (entry.target.window_ref === ref && (!kind || observationKind(entry.result) === kind)) this.snapshots.delete(id);}private remember(target: ObservationTarget, result: ComputerResult): ComputerResult {const id = result.details.observation_id ?? result.details.agent_image?.image_id;// A delivered DOM action carries the page's next snapshot, which becomes the current observation.const observed = result.details.status === "observed" || result.details.status === "delivered" && result.details.dom !== undefined;if (!observed || typeof id !== "string") return result;- for (const [old, entry] of this.snapshots) if (sameTarget(entry.target, target)) this.snapshots.delete(old);+ // A window keeps one DOM snapshot and one native capture: a new one replaces only the earlier one of its kind.+ for (const [old, entry] of this.snapshots) if (sameTarget(entry.target, target) && observationKind(entry.result) === observationKind(result)) this.snapshots.delete(old);this.snapshots.set(id, { target, result });if (target.window_ref !== undefined && result.details.ui) {this.lastUi.delete(target.window_ref);@@ -733,8 +737,8 @@ export class ComputerResources {}if (view.kind === "frames") throw new ComputerRequestError("Frame selection needs a sequence address.");if (address.kind === "window") {- // Fresh window Read retires both provider handles before any native acquisition.- this.retireWindow(address.ref);+ // A fresh window Read retires the earlier observation of its kind; a DOM snapshot and a native capture coexist.+ this.retireWindow(address.ref, view.kind === "dom" ? "dom" : view.kind === "state" ? undefined : "native");if (view.kind === "state") {const result = await session.observeWindowState(address.ref, signal);if (result.details.status === "observed" && result.details.window_state_id) {@@ -766,7 +770,7 @@ export class ComputerResources {return session.webImage(ref, address.ref, view.operation, { ref: view.hint }, signal);}if (view.kind === "dom" && Object.keys(view.options).length) {- this.retireWindow(ref);+ this.retireWindow(ref, "dom");return this.remember({ window_ref: ref }, await session.observeDom(ref, view.options, signal));}if (view.kind === "overview" && view.capture.granularity === "image" && Object.keys(view.capture).length === 1)diff --git a/src/session.ts b/src/session.tsindex fb91dd9..1d1b142 100644--- a/src/session.ts+++ b/src/session.ts@@ -975,7 +975,8 @@ export class ComputerSession {}private async capture(entry: WindowEntry, signal?: AbortSignal, query?: string, options: UiCaptureOptions = {}, quiet = false): Promise<ComputerResult> {validateUiCapture(options);- entry.dom = undefined; entry.windowState = undefined;+ // A native capture keeps the page's DOM snapshot: reading changes nothing, so both stay current until input.+ entry.windowState = undefined;const granularity = options.granularity ?? "full";const hasUi = granularity !== "image"; const hasImage = granularity !== "ui";if (!hasUi && query !== undefined) return this.result("refused", "Image-only observation cannot query UI labels.");@@ -1111,7 +1112,8 @@ export class ComputerSession {return this.queue(async () => {const entry = await this.webEntry(ref, signal);if (!("ref" in entry)) return entry;- entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;+ // A DOM snapshot keeps the window's native capture: reading changes nothing, so both stay current until input.+ entry.dom = undefined; entry.windowState = undefined;try {const observed = this.webObservation(entry, await entry.web!.page("snapshot", options, signal));return this.result("observed", "Fresh Playwright aria snapshot. Use [ref=...] values as target.ref while this stays the latest snapshot of the window.",@@ -1704,7 +1706,8 @@ export class ComputerSession {if (signal?.aborted) return finish(this.result("refused", "Cancelled during action mark; input was not dispatched."), "Cancelled during action mark.");if (!this.allowed(entry)) return finish(this.denied(entry.ref), "Permission revoked during action mark.");if (batch && performance.now() - started >= SEQUENCE_ELAPSED_MS) return finish(this.result("refused", "Sequence time budget reached."), `${SEQUENCE_ELAPSED_MS / 60000} min elapsed budget reached during admission; no more steps dispatched.`);- entry.observation = undefined; entry.detail = undefined;+ // Input may change the page, so the window's DOM snapshot retires with its native capture.+ entry.observation = undefined; entry.detail = undefined; entry.dom = undefined;const pointerAction = isPointerMove(step.action) || isCoordinateScroll(step) || ["click", "right_click", "double_click", "drag"].includes(step.action) || (step.action === "hold" && !!step.actions?.length);if (pointerAction) entry.lastAction = { reason: "Pointer outcome is not yet confirmed." };const requested = { ...step, ...(step.text !== undefined ? { text_length: step.text.length } : {}),diff --git a/test/live/typed-input.live.ts b/test/live/typed-input.live.tsindex e638f66..96e6466 100644--- a/test/live/typed-input.live.ts+++ b/test/live/typed-input.live.ts@@ -66,6 +66,27 @@ test("one action vocabulary drives a native window by locator, ref and point", a} finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }});+test("points of a native canvas draw strokes that the receipt shows as changed pixels", async () => {+ const opened = await live.insert(live.machine, { action: "open", app: "Paint" });+ expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");+ const window = `computer:window/${opened.details.window_ref}`;+ try {+ const image = await live.read(window, ["image"]);+ const size = image.details.state as { screenshot_width: number; screenshot_height: number };+ const middle = { x: Math.round(size.screenshot_width / 2), y: Math.round(size.screenshot_height * 0.6) };+ const stroke = await act(window, dragPlan({ window, from: { x: middle.x - 120, y: middle.y }, to: { x: middle.x + 120, y: middle.y + 40 }, via: [{ x: middle.x, y: middle.y - 60 }] }));+ expect(stroke.result.details.status, stroke.body).toBe("delivered");+ expect(stroke.body).toMatch(/Pixels changed in \d+ area/);+ const line = await act(window, timelinePlan({ window, events: [{ at_ms: 0, type: "move", x: middle.x - 100, y: middle.y + 80 }, { at_ms: 20, type: "button_down" },+ { at_ms: 60, type: "move", x: middle.x, y: middle.y + 100 }, { at_ms: 100, type: "move", x: middle.x + 100, y: middle.y + 80 }, { at_ms: 140, type: "button_up" }] }));+ expect(line.result.details.status, line.body).toBe("delivered");+ expect(line.body).toMatch(/Pixels changed in \d+ area/);+ } finally {+ // Paint asks about the unsaved drawing, so the window the session opened is ended without saving.+ await deleteComputer(live.resources, `${window}?force=true`, { path: `${window}?force=true` }, live.cwd);+ }+});+test("points of a browser page take clicks, wheel, drags, held keys and a pressure pen through Playwright", async () => {const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/input` });expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");@@ -100,8 +121,14 @@ test("the same vocabulary drives a browser page through Playwright by locator anconst window = `computer:window/${opened.details.window_ref}`;try {expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Ada" }))).result.details.status).toBe("delivered");+ // A native Read and a DOM Read of one window keep each other: the browser's own Reload button stays a usable ref.+ const reload = /button "Reload" \[ref=([0-9a-z]+:[a-z]+)\]/.exec(presentComputerResult(await live.read(window, ["aria"])))![1];+ expect((await live.read(window, ["dom"])).details.status).toBe("observed");+ const reloaded = await act(window, clickPlan({ window, target: reload }));+ expect(reloaded.result.details.status, reloaded.body).toBe("delivered");+ expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Ada", replace: true }))).result.details.status).toBe("delivered");const dom = presentComputerResult(await live.read(window, ["dom"]));- const save = /button "Save" \[ref=(e\d+)\]/.exec(dom)![1];+ const save = /button "Save" \[ref=((?:f\d+)?e\d+)\]/.exec(dom)![1];const clicked = await act(window, clickPlan({ window, target: save }));expect(clicked.result.details.status, clicked.body).toBe("delivered");expect(await holds(window, [{ target: { text: "saved Ada" }, is: "visible" }])).toBe("satisfied");