Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Pin runtime 2f1a8b1d: a press on a covered point refuses and names the covering window (LPT-955)

alexshpunt-clanker[bot]fb91e36910aa
diff --git a/src/driver.ts b/src/driver.ts
index dc25e31..e67d3c9 100644
--- a/src/driver.ts
+++ b/src/driver.ts
@@ -86,8 +86,8 @@ export const WINDOW_RESTORE_RUNTIME = {
} as const;
/** Window management plus every earlier native capability; `windows-desktop` selects these bytes. Browser pages use Playwright, not this runtime. */
export const WINDOW_MANAGEMENT_RUNTIME = {
- source: "c920b66bd60d064aa0d9f5662e9b30f1648222fd", version: "0.37.0",
- sha256: "7a633781120a6648dca16a332cc302f42e0137db5f874e1d1dd7d4d17d727120",
+ source: "2f1a8b1ddf1fb640c6d5da4ed870c1c2306ff92e", version: "0.37.0",
+ sha256: "3c64b031bf8f67dba5914e963f069ac31c6d72f2d14af8e46cf6dd1e1a6f8c00",
} as const;
const forkRuntimes = {
"windows-desktop": WINDOW_MANAGEMENT_RUNTIME,
diff --git a/src/runtime-pins.json b/src/runtime-pins.json
index 8fe69ab..40b357a 100644
--- a/src/runtime-pins.json
+++ b/src/runtime-pins.json
@@ -1,12 +1,12 @@
{
"format": 2,
- "source": "c920b66bd60d064aa0d9f5662e9b30f1648222fd",
+ "source": "2f1a8b1ddf1fb640c6d5da4ed870c1c2306ff92e",
"version": "0.37.0",
- "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-c920b66b/",
+ "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-2f1a8b1d/",
"files": {
- "cua-driver.exe": "7a633781120a6648dca16a332cc302f42e0137db5f874e1d1dd7d4d17d727120",
- "cua-driver-uia.exe": "7789da185c8d4c62da8adf245825311e48f914880580c996d7c30552917159d6",
- "cua-cursor-theme.exe": "9ca25534dd2d2c95088521e0547863482f05e37591ff699531796dd5f3a433b0",
+ "cua-driver.exe": "3c64b031bf8f67dba5914e963f069ac31c6d72f2d14af8e46cf6dd1e1a6f8c00",
+ "cua-driver-uia.exe": "2d2ae2097ef6f1514b2fcac04cdd8643b8a605d76f04a87fd865ebf999e3dea4",
+ "cua-cursor-theme.exe": "8447ceff285619777dd4921d58caae196408ba419d29230876e1447621adf430",
"LICENSE.md": "d0a32419a44fa38d5023d1431dfcbedd1457eb5e7b2e6c7c87ca064facb41b41",
"THIRD_PARTY_NOTICES.md": "88a24f59293f03b3e03655e8144bcfadc9861bef1182cfeea91639e11f573fd1"
}
diff --git a/src/session.ts b/src/session.ts
index dccd8a4..8202de8 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -26,6 +26,7 @@ import { setTimeout as delay } from "node:timers/promises";
import { cropDisplay, nativeRegion, validateDetail, type DetailContext, type DetailRequest } from "./native-detail.ts";
import type { StateDiff } from "./successor.ts";
import { combine, evaluateNative, validateWait, type ConditionResult, type WaitResult, type WaitSpec } from "./ui-conditions.ts";
+import { computerSource } from "./computer-address.ts";
import { WebError, type WebAction, type WebBrowser, type WebLauncher, type WebLaunchOptions, type WebPage, type WebSnapshotOptions, type WebTarget } from "./web-browser.ts";
/** Native Windows address. Models use session window references instead. */
@@ -192,6 +193,16 @@ const appName = (name: string) => name.trim().toLowerCase().replace(/^windows\s+
/** File name of a Win32 launch path without directory and extension; packaged shell:appsFolder paths have none. */
const executableName = (path: string) => /^shell:/i.test(path) ? "" : appName(path.split(/[\\/]/).pop() ?? "");
+interface CoveringWindow { pid: number; window_id: number; relation: string; title: string }
+/** The window a driver refusal names as covering the press point (code pointer_target_occluded). */
+function coveringWindow(message: string): CoveringWindow | undefined {
+ const match = /pointer_target_occluded: .*?window_id=(\d+) pid=(\d+) relation=(\w+) title=("(?:[^"\\]|\\.)*")/.exec(message);
+ if (!match) return undefined;
+ let title = "";
+ try { title = String(JSON.parse(match[4])); } catch { /* A malformed title leaves the window untitled. */ }
+ return { window_id: Number(match[1]), pid: Number(match[2]), relation: match[3], title };
+}
+
/** Multi-window UI policy. Native work is queued; ownership and observations are not persisted across reloads. */
export class ComputerSession {
private tail: Promise<unknown> = Promise.resolve();
@@ -1942,9 +1953,12 @@ export class ComputerSession {
const nativeMessage = native.content.filter(c => c.type === "text").map(c => c.text).join("\n");
// Foreground input checks for recent physical input before sending anything (CUA_HUMAN_IDLE_MS).
const takeover = /human_takeover:/.test(nativeMessage) && (native.isError || state.effect === "refused");
+ // A real press at a point another window covers is refused by the driver before anything is pressed there.
+ const covering = coveringWindow(nativeMessage);
const refused = takeover || state.status === "refused" || state.effect === "refused" || ["invalid_action_target", "invalid_arguments", "stale_element_token",
"conflicting_element_target", "background_unavailable", "background_occluded", "background_uipi_blocked", "key_hold_unavailable", "held_path_unavailable"].includes(String(state.code));
- const uncertain = native.isError || ["unknown", "partial"].includes(String(state.status)) || ["unknown", "partial"].includes(String(state.effect));
+ // A covered press that is not a refusal stopped an action partway, after earlier events were sent.
+ const uncertain = native.isError || !!covering || ["unknown", "partial"].includes(String(state.status)) || ["unknown", "partial"].includes(String(state.effect));
const inconsistent = !refused && !uncertain && (isSemanticAction(step.action) ? !semanticReceipt(step.action, state)
: isPointerMove(step.action) ? delivery_mode !== "foreground" || !pointerReceipt(step, state)
: isCoordinateScroll(step) && (delivery_mode !== "foreground" || !scrollReceipt(step, state)));
@@ -1975,7 +1989,9 @@ export class ComputerSession {
entry.lastAction = { reason: "In-flight pointer outcome was withheld after revocation or cancellation." };
return finish(this.result("unknown", "Input was dispatched before revocation or cancellation. Do not replay blindly."), "In-flight outcome withheld after input.");
}
- const result = this.result(status, takeover ? `Foreground input paused because the human is using the computer: ${nativeMessage.slice(nativeMessage.indexOf("human_takeover:") + 16)}` : `Driver returned ${status} for ${name} (requested: ${delivery_mode ?? "background"}). ${isSemanticAction(step.action) ? "A UIA receipt is not app or focus verification. " : ""}Observe again to verify application state.${step.action === "hold" && typeof state.summary === "string" ? `\n${state.summary}` : ""}`,
+ const cover = covering && await this.coverText(covering, signal);
+ const result = this.result(status, takeover ? `Foreground input paused because the human is using the computer: ${nativeMessage.slice(nativeMessage.indexOf("human_takeover:") + 16)}`
+ : cover && status === "refused" ? `Nothing was pressed: ${cover}` : `${cover ? `A press stopped: ${cover}\n` : ""}Driver returned ${status} for ${name} (requested: ${delivery_mode ?? "background"}). ${isSemanticAction(step.action) ? "A UIA receipt is not app or focus verification. " : ""}Observe again to verify application state.${step.action === "hold" && typeof state.summary === "string" ? `\n${state.summary}` : ""}`,
{ ...state, ...(step.element_token || pointerAction ? { coordinate_grounding: { input_space: step.element_token ? "native_element_token" : "window_image_pixels", transform: "cua_native_once",
overview: { width: observed.width, height: observed.height }, ...(step.x !== undefined ? { requested: { x: step.x, y: step.y } } : {}),
resolved_native_screen: resolvedPoint ?? null, native_target_mark: isSemanticAction(step.action) ? null : entry.lastAction ?? null } } : {}),
@@ -1997,6 +2013,16 @@ export class ComputerSession {
});
}
+ /** Names the window that covers a press point, with its address where this session may act on it. */
+ private async coverText(covering: CoveringWindow, signal?: AbortSignal): Promise<string> {
+ const entry = this.getMode() === "yolo" ? await this.register(covering, signal).catch(() => undefined) : undefined;
+ const named = `${covering.title ? `"${covering.title}"` : "an untitled window"}${entry ? ` (${computerSource("window", entry.ref)})` : ""}`;
+ const then = "Close or move it, or act on it instead.";
+ if (covering.relation === "owned_window")
+ return `${named}, a window this window owns, covers the point. It stays in front of this window even when this window is active. ${then}`;
+ return `${covering.relation === "same_process" ? "another window of this app" : "a window of another app"}, ${named}, covers the point. ${then}`;
+ }
+
/** Why a kept pixel observation no longer matches its window, or undefined while it still does. Movement keeps window-local points valid. */
private async observationChange(entry: WindowEntry, observed: Observation, signal?: AbortSignal): Promise<string | undefined> {
const native = await this.driver.call("get_window_control_state", { pid: entry.target.pid, window_id: entry.target.window_id }, signal);