Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Pin runtime d9f380f6: a click that closes its dialog is delivered and names the app's window in front (LPT-954)
alexshpunt-clanker[bot]3052fa72e9a0
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex a621ef6..2a4e4b8 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -1377,10 +1377,16 @@ export class ComputerResources {crops = await cropImages(follow.page, shot, diff);}}+ // A window that could not be read again may have closed, as a dialog does after OK: say so and name the app's window+ // in front. The receipt already names it when the driver saw the click close its window.+ const target = before.result.details.target ?? result.details.target;+ const closed = after?.observation_id || status === "refused" || !target ? undefined+ : (result.details.state as { foreground_after?: unknown } | undefined)?.foreground_after ? "This window closed or is hidden."+ : await session.closedWindowNote(target, signal).catch(() => undefined);// The successor replaces the action's own observation as the current one; a wait retired the earlier snapshot.const details = { ...result.details, ...(after?.observation_id ? { observation_id: after.observation_id, ...(after.dom ? { dom: after.dom } : {}) } : {}),successor: after?.observation_id ? { observation_id: after.observation_id, ...(diff ? { diff } : { reason: "The input observation had no comparable UI or DOM." }) }- : { reason: status === "refused" ? "The action was refused; the input observation is unchanged but consumed." : "No successor observation could be taken." } };+ : { reason: status === "refused" ? "The action was refused; the input observation is unchanged but consumed." : closed ?? "No successor observation could be taken." } };return { ...result, content: [...result.content, ...crops], details };}diff --git a/src/driver.ts b/src/driver.tsindex e67d3c9..f17278d 100644--- a/src/driver.ts+++ b/src/driver.ts@@ -86,8 +86,8 @@ export const WINDOW_RESTORE_RUNTIME = {} as const;/** Window management plus every earlier native capability; `windows-desktop` selects these bytes. Browser pages use Playwright, not this runtime. */export const WINDOW_MANAGEMENT_RUNTIME = {- source: "2f1a8b1ddf1fb640c6d5da4ed870c1c2306ff92e", version: "0.37.0",- sha256: "3c64b031bf8f67dba5914e963f069ac31c6d72f2d14af8e46cf6dd1e1a6f8c00",+ source: "d9f380f68550691eec806dc45fdbc60f6d066de6", version: "0.37.0",+ sha256: "057eecd1150fa462d9838e7475a35690d29a05fb2783ed1c91083a2605a973eb",} as const;const forkRuntimes = {"windows-desktop": WINDOW_MANAGEMENT_RUNTIME,diff --git a/src/provider.ts b/src/provider.tsindex f6b595f..9ab1830 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -193,7 +193,8 @@ export function presentComputerResult(result: ComputerResult, full = false): strconst s = d.successor, diff = s.diff;lines.push(s.observation_id ? `Successor: ${computerSource("observation", s.observation_id)}` : `Successor: none. ${s.reason ?? ""}`);if (diff) lines.push(...diffLines(diff, true));- else if (s.reason) lines.push(s.reason);+ // Without a successor, the reason already follows "Successor: none".+ else if (s.reason && s.observation_id) lines.push(s.reason);}if (d.selected_frames) lines.push(`Selected frames: ${d.selected_frames.map(frame => frame.index).join(", ")}; readonly, input=none`);lines.push(d.successor?.observation_id ? "Refs in this change list are current for the next input; Read the window for its full state."diff --git a/src/runtime-pins.json b/src/runtime-pins.jsonindex 40b357a..0b359bf 100644--- a/src/runtime-pins.json+++ b/src/runtime-pins.json@@ -1,12 +1,12 @@{"format": 2,- "source": "2f1a8b1ddf1fb640c6d5da4ed870c1c2306ff92e",+ "source": "d9f380f68550691eec806dc45fdbc60f6d066de6","version": "0.37.0",- "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-2f1a8b1d/",+ "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-d9f380f6/","files": {- "cua-driver.exe": "3c64b031bf8f67dba5914e963f069ac31c6d72f2d14af8e46cf6dd1e1a6f8c00",- "cua-driver-uia.exe": "2d2ae2097ef6f1514b2fcac04cdd8643b8a605d76f04a87fd865ebf999e3dea4",- "cua-cursor-theme.exe": "8447ceff285619777dd4921d58caae196408ba419d29230876e1447621adf430",+ "cua-driver.exe": "057eecd1150fa462d9838e7475a35690d29a05fb2783ed1c91083a2605a973eb",+ "cua-driver-uia.exe": "4da50c45a51a205c65625d352a95a6a937e914ac51426c81f439d128d8e2cf5d",+ "cua-cursor-theme.exe": "bd201820ed990ed6b590bb289387138d9c0411649817c4870c4cf58e306dba4e","LICENSE.md": "d0a32419a44fa38d5023d1431dfcbedd1457eb5e7b2e6c7c87ca064facb41b41","THIRD_PARTY_NOTICES.md": "88a24f59293f03b3e03655e8144bcfadc9861bef1182cfeea91639e11f573fd1"}diff --git a/src/session.ts b/src/session.tsindex 8202de8..e4555e0 100644--- a/src/session.ts+++ b/src/session.ts@@ -193,7 +193,14 @@ const appName = (name: string) => name.trim().toLowerCase().replace(/^windows\s+/** File name of a Win32 launch path without directory and extension; packaged shell:appsFolder paths have none. */const executableName = (path: string) => /^shell:/i.test(path) ? "" : appName(path.split(/[\\/]/).pop() ?? "");-interface CoveringWindow { pid: number; window_id: number; relation: string; title: string }+interface NamedWindow { pid: number; window_id: number; title: string }+interface CoveringWindow extends NamedWindow { relation: string }+/** The window a click result names as in front after the click closed its own window. */+function foregroundAfter(value: unknown): NamedWindow | undefined {+ const window = value as Partial<NamedWindow> | undefined;+ return window && Number.isSafeInteger(window.pid) && Number.isSafeInteger(window.window_id)+ ? { pid: window.pid!, window_id: window.window_id!, title: typeof window.title === "string" ? window.title : "" } : undefined;+}/** The window a driver refusal names as covering the press point (code pointer_target_occluded). */function coveringWindow(message: string): CoveringWindow | undefined {const match = /pointer_target_occluded: .*?window_id=(\d+) pid=(\d+) relation=(\w+) title=("(?:[^"\\]|\\.)*")/.exec(message);@@ -1990,8 +1997,11 @@ export class ComputerSession {return finish(this.result("unknown", "Input was dispatched before revocation or cancellation. Do not replay blindly."), "In-flight outcome withheld after input.");}const cover = covering && await this.coverText(covering, signal);+ // A click that closed its window, such as OK in a dialog, names the window of the app the driver found in front.+ const after = status === "delivered" ? foregroundAfter(state.foreground_after) : undefined;+ const front = after && `The click closed this window; ${await this.windowName(after, signal)} of the same app is now in front.\n`;const result = this.result(status, takeover ? `Foreground input paused because the human is using the computer: ${nativeMessage.slice(nativeMessage.indexOf("human_takeover:") + 16)}`- : cover && status === "refused" ? `Nothing was pressed: ${cover}` : `${cover ? `A press stopped: ${cover}\n` : ""}Driver returned ${status} for ${name} (requested: ${delivery_mode ?? "background"}). ${isSemanticAction(step.action) ? "A UIA receipt is not app or focus verification. " : ""}Observe again to verify application state.${step.action === "hold" && typeof state.summary === "string" ? `\n${state.summary}` : ""}`,+ : cover && status === "refused" ? `Nothing was pressed: ${cover}` : `${cover ? `A press stopped: ${cover}\n` : ""}${front ?? ""}Driver returned ${status} for ${name} (requested: ${delivery_mode ?? "background"}). ${isSemanticAction(step.action) ? "A UIA receipt is not app or focus verification. " : ""}Observe again to verify application state.${step.action === "hold" && typeof state.summary === "string" ? `\n${state.summary}` : ""}`,{ ...state, ...(step.element_token || pointerAction ? { coordinate_grounding: { input_space: step.element_token ? "native_element_token" : "window_image_pixels", transform: "cua_native_once",overview: { width: observed.width, height: observed.height }, ...(step.x !== undefined ? { requested: { x: step.x, y: step.y } } : {}),resolved_native_screen: resolvedPoint ?? null, native_target_mark: isSemanticAction(step.action) ? null : entry.lastAction ?? null } } : {}),@@ -2013,10 +2023,31 @@ export class ComputerSession {});}+ /**+ * After input whose window could not be read again: when the window is gone from the window list (closed or hidden),+ * say so and name the app's frontmost remaining window. The list is in the desktop's z-order, front first. It takes+ * the native target, since a closed window's reference is already dropped.+ */+ closedWindowNote(target: WindowTarget, signal?: AbortSignal): Promise<string | undefined> {+ return this.queue(async () => {+ if (this.driver.windowLifetime === false) return undefined;+ const windows = await this.list(signal).catch(() => undefined);+ if (!windows || windows.some(window => window.pid === target.pid && window.window_id === target.window_id)) return undefined;+ const front = windows.find(window => window.pid === target.pid && Number.isSafeInteger(window.window_id));+ return `This window closed or is hidden; ${front ? `the app's frontmost window is ${await this.windowName({ pid: target.pid,+ window_id: front.window_id as number, title: typeof front.title === "string" ? front.title : "" }, signal)}` : "the app has no other visible window"}.`;+ });+ }++ /** A window the driver named by its handle: its title, and its address where this session may act on it. YOLO adopts it; Safe never does. */+ private async windowName(window: NamedWindow, signal?: AbortSignal): Promise<string> {+ const entry = this.getMode() === "yolo" ? await this.register(window, signal).catch(() => undefined) : undefined;+ return `${window.title ? `"${window.title}"` : "an untitled window"}${entry ? ` (${computerSource("window", entry.ref)})` : ""}`;+ }+/** Names the window that covers a press point, with its address where this session may act on it. */private async coverText(covering: CoveringWindow, signal?: AbortSignal): Promise<string> {- const entry = this.getMode() === "yolo" ? await this.register(covering, signal).catch(() => undefined) : undefined;- const named = `${covering.title ? `"${covering.title}"` : "an untitled window"}${entry ? ` (${computerSource("window", entry.ref)})` : ""}`;+ const named = await this.windowName(covering, signal);const then = "Close or move it, or act on it instead.";if (covering.relation === "owned_window")return `${named}, a window this window owns, covers the point. It stays in front of this window even when this window is active. ${then}`;