Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Pin runtime 2f1a8b1d: a press on a covered point refuses and names the covering window (LPT-955)
alexshpunt-clanker[bot]fb91e36910aa
diff --git a/src/driver.ts b/src/driver.tsindex dc25e31..e67d3c9 100644--- a/src/driver.ts+++ b/src/driver.ts@@ -86,8 +86,8 @@ export const WINDOW_RESTORE_RUNTIME = {} as const;/** Window management plus every earlier native capability; `windows-desktop` selects these bytes. Browser pages use Playwright, not this runtime. */export const WINDOW_MANAGEMENT_RUNTIME = {- source: "c920b66bd60d064aa0d9f5662e9b30f1648222fd", version: "0.37.0",- sha256: "7a633781120a6648dca16a332cc302f42e0137db5f874e1d1dd7d4d17d727120",+ source: "2f1a8b1ddf1fb640c6d5da4ed870c1c2306ff92e", version: "0.37.0",+ sha256: "3c64b031bf8f67dba5914e963f069ac31c6d72f2d14af8e46cf6dd1e1a6f8c00",} as const;const forkRuntimes = {"windows-desktop": WINDOW_MANAGEMENT_RUNTIME,diff --git a/src/runtime-pins.json b/src/runtime-pins.jsonindex 8fe69ab..40b357a 100644--- a/src/runtime-pins.json+++ b/src/runtime-pins.json@@ -1,12 +1,12 @@{"format": 2,- "source": "c920b66bd60d064aa0d9f5662e9b30f1648222fd",+ "source": "2f1a8b1ddf1fb640c6d5da4ed870c1c2306ff92e","version": "0.37.0",- "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-c920b66b/",+ "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-2f1a8b1d/","files": {- "cua-driver.exe": "7a633781120a6648dca16a332cc302f42e0137db5f874e1d1dd7d4d17d727120",- "cua-driver-uia.exe": "7789da185c8d4c62da8adf245825311e48f914880580c996d7c30552917159d6",- "cua-cursor-theme.exe": "9ca25534dd2d2c95088521e0547863482f05e37591ff699531796dd5f3a433b0",+ "cua-driver.exe": "3c64b031bf8f67dba5914e963f069ac31c6d72f2d14af8e46cf6dd1e1a6f8c00",+ "cua-driver-uia.exe": "2d2ae2097ef6f1514b2fcac04cdd8643b8a605d76f04a87fd865ebf999e3dea4",+ "cua-cursor-theme.exe": "8447ceff285619777dd4921d58caae196408ba419d29230876e1447621adf430","LICENSE.md": "d0a32419a44fa38d5023d1431dfcbedd1457eb5e7b2e6c7c87ca064facb41b41","THIRD_PARTY_NOTICES.md": "88a24f59293f03b3e03655e8144bcfadc9861bef1182cfeea91639e11f573fd1"}diff --git a/src/session.ts b/src/session.tsindex dccd8a4..8202de8 100644--- a/src/session.ts+++ b/src/session.ts@@ -26,6 +26,7 @@ import { setTimeout as delay } from "node:timers/promises";import { cropDisplay, nativeRegion, validateDetail, type DetailContext, type DetailRequest } from "./native-detail.ts";import type { StateDiff } from "./successor.ts";import { combine, evaluateNative, validateWait, type ConditionResult, type WaitResult, type WaitSpec } from "./ui-conditions.ts";+import { computerSource } from "./computer-address.ts";import { WebError, type WebAction, type WebBrowser, type WebLauncher, type WebLaunchOptions, type WebPage, type WebSnapshotOptions, type WebTarget } from "./web-browser.ts";/** Native Windows address. Models use session window references instead. */@@ -192,6 +193,16 @@ const appName = (name: string) => name.trim().toLowerCase().replace(/^windows\s+/** File name of a Win32 launch path without directory and extension; packaged shell:appsFolder paths have none. */const executableName = (path: string) => /^shell:/i.test(path) ? "" : appName(path.split(/[\\/]/).pop() ?? "");+interface CoveringWindow { pid: number; window_id: number; relation: string; title: string }+/** The window a driver refusal names as covering the press point (code pointer_target_occluded). */+function coveringWindow(message: string): CoveringWindow | undefined {+ const match = /pointer_target_occluded: .*?window_id=(\d+) pid=(\d+) relation=(\w+) title=("(?:[^"\\]|\\.)*")/.exec(message);+ if (!match) return undefined;+ let title = "";+ try { title = String(JSON.parse(match[4])); } catch { /* A malformed title leaves the window untitled. */ }+ return { window_id: Number(match[1]), pid: Number(match[2]), relation: match[3], title };+}+/** Multi-window UI policy. Native work is queued; ownership and observations are not persisted across reloads. */export class ComputerSession {private tail: Promise<unknown> = Promise.resolve();@@ -1942,9 +1953,12 @@ export class ComputerSession {const nativeMessage = native.content.filter(c => c.type === "text").map(c => c.text).join("\n");// Foreground input checks for recent physical input before sending anything (CUA_HUMAN_IDLE_MS).const takeover = /human_takeover:/.test(nativeMessage) && (native.isError || state.effect === "refused");+ // A real press at a point another window covers is refused by the driver before anything is pressed there.+ const covering = coveringWindow(nativeMessage);const refused = takeover || state.status === "refused" || state.effect === "refused" || ["invalid_action_target", "invalid_arguments", "stale_element_token","conflicting_element_target", "background_unavailable", "background_occluded", "background_uipi_blocked", "key_hold_unavailable", "held_path_unavailable"].includes(String(state.code));- const uncertain = native.isError || ["unknown", "partial"].includes(String(state.status)) || ["unknown", "partial"].includes(String(state.effect));+ // A covered press that is not a refusal stopped an action partway, after earlier events were sent.+ const uncertain = native.isError || !!covering || ["unknown", "partial"].includes(String(state.status)) || ["unknown", "partial"].includes(String(state.effect));const inconsistent = !refused && !uncertain && (isSemanticAction(step.action) ? !semanticReceipt(step.action, state): isPointerMove(step.action) ? delivery_mode !== "foreground" || !pointerReceipt(step, state): isCoordinateScroll(step) && (delivery_mode !== "foreground" || !scrollReceipt(step, state)));@@ -1975,7 +1989,9 @@ export class ComputerSession {entry.lastAction = { reason: "In-flight pointer outcome was withheld after revocation or cancellation." };return finish(this.result("unknown", "Input was dispatched before revocation or cancellation. Do not replay blindly."), "In-flight outcome withheld after input.");}- const result = this.result(status, takeover ? `Foreground input paused because the human is using the computer: ${nativeMessage.slice(nativeMessage.indexOf("human_takeover:") + 16)}` : `Driver returned ${status} for ${name} (requested: ${delivery_mode ?? "background"}). ${isSemanticAction(step.action) ? "A UIA receipt is not app or focus verification. " : ""}Observe again to verify application state.${step.action === "hold" && typeof state.summary === "string" ? `\n${state.summary}` : ""}`,+ const cover = covering && await this.coverText(covering, signal);+ const result = this.result(status, takeover ? `Foreground input paused because the human is using the computer: ${nativeMessage.slice(nativeMessage.indexOf("human_takeover:") + 16)}`+ : cover && status === "refused" ? `Nothing was pressed: ${cover}` : `${cover ? `A press stopped: ${cover}\n` : ""}Driver returned ${status} for ${name} (requested: ${delivery_mode ?? "background"}). ${isSemanticAction(step.action) ? "A UIA receipt is not app or focus verification. " : ""}Observe again to verify application state.${step.action === "hold" && typeof state.summary === "string" ? `\n${state.summary}` : ""}`,{ ...state, ...(step.element_token || pointerAction ? { coordinate_grounding: { input_space: step.element_token ? "native_element_token" : "window_image_pixels", transform: "cua_native_once",overview: { width: observed.width, height: observed.height }, ...(step.x !== undefined ? { requested: { x: step.x, y: step.y } } : {}),resolved_native_screen: resolvedPoint ?? null, native_target_mark: isSemanticAction(step.action) ? null : entry.lastAction ?? null } } : {}),@@ -1997,6 +2013,16 @@ export class ComputerSession {});}+ /** Names the window that covers a press point, with its address where this session may act on it. */+ private async coverText(covering: CoveringWindow, signal?: AbortSignal): Promise<string> {+ const entry = this.getMode() === "yolo" ? await this.register(covering, signal).catch(() => undefined) : undefined;+ const named = `${covering.title ? `"${covering.title}"` : "an untitled window"}${entry ? ` (${computerSource("window", entry.ref)})` : ""}`;+ const then = "Close or move it, or act on it instead.";+ if (covering.relation === "owned_window")+ return `${named}, a window this window owns, covers the point. It stays in front of this window even when this window is active. ${then}`;+ return `${covering.relation === "same_process" ? "another window of this app" : "a window of another app"}, ${named}, covers the point. ${then}`;+ }+/** Why a kept pixel observation no longer matches its window, or undefined while it still does. Movement keeps window-local points valid. */private async observationChange(entry: WindowEntry, observed: Observation, signal?: AbortSignal): Promise<string | undefined> {const native = await this.driver.call("get_window_control_state", { pid: entry.target.pid, window_id: entry.target.window_id }, signal);