Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
fix: preserve unknown verification results on cancellation
alexshpunt-clanker[bot]24beaf3c31ff
diff --git a/src/session.ts b/src/session.tsindex 3484641..e882ba1 100644--- a/src/session.ts+++ b/src/session.ts@@ -1511,12 +1511,17 @@ export class ComputerSession {let samples = 0, stable = 0, results: ConditionResult[] = [], last: ComputerResult | undefined;entry.dom = undefined; entry.windowState = undefined;const finish = async (verdict: WaitResult["verdict"], timedOut: boolean, final: ComputerResult | undefined, why?: string): Promise<ComputerResult> => {+ if (!this.allowed(entry!)) return this.denied(entry!.ref);+ await this.activity(entry!, { kind: "verify" }, signal, undefined, verdict === "satisfied" ? "verified" : "unknown");+ if (!this.allowed(entry!)) return this.denied(entry!.ref);+ if (signal?.aborted) {+ verdict = "unknown";+ why = "Cancelled; the last sample does not decide the conditions.";+ }const verification = { verdict, conditions: results, samples, elapsed_ms: Math.round(performance.now() - started), timed_out: timedOut,...(why ? { reason: why } : {}), spec };const message = `Verification ${verdict} after ${samples} sample(s) in ${verification.elapsed_ms} ms${timedOut ? " (timed out)" : ""}.${why ? ` ${why}` : ""}`;const d = final?.details;- await this.activity(entry!, { kind: "verify" }, signal, undefined, verdict === "satisfied" ? "verified" : "unknown");- if (!this.allowed(entry!) || signal?.aborted) return this.result("refused", "Verification revoked or cancelled; data withheld.");return this.result("observed", message, undefined, { ...this.addressed(entry!), verification,...(d?.observation_id ? { observation_id: d.observation_id } : {}), ...(d?.ui ? { ui: d.ui } : {}), ...(d?.dom ? { dom: d.dom } : {}) });};diff --git a/test/window-observer.test.ts b/test/window-observer.test.tsindex 0f1daa1..feb39b9 100644--- a/test/window-observer.test.ts+++ b/test/window-observer.test.ts@@ -20,7 +20,7 @@ async function fixture(observer: WindowObserver) {const created = { pid: 42, window_id: 101, app_name: "Calculator.exe", title: "Owned", bounds };windows.push(created); return { content: [], structuredContent: { pid: 42, windows: [created] } };}- if (name === "get_window_state") return { content: [args.max_image_dimension === 4096 ? raw : png], structuredContent: { ...args, elements: [{ element_token: "root", frame: { x: 30, y: 40, w: 1000, h: 700 }, screenshot_frame: { x: -1, y: -1, w: 500, h: 350 } }, { element_token: "button", frame: { x: 230, y: 240, w: 120, h: 60 }, screenshot_frame: { x: 100, y: 100, w: 60, h: 30 } }], screenshot_width: args.max_image_dimension === 4096 ? 1000 : 500, screenshot_height: args.max_image_dimension === 4096 ? 700 : 350, window_bounds: bounds } };+ if (name === "get_window_state") return { content: [args.max_image_dimension === 4096 ? raw : png], structuredContent: { ...args, elements: [{ element_token: "root", role: "Window", frame: { x: 30, y: 40, w: 1000, h: 700 }, screenshot_frame: { x: -1, y: -1, w: 500, h: 350 } }, { element_token: "button", role: "Button", frame: { x: 230, y: 240, w: 120, h: 60 }, screenshot_frame: { x: 100, y: 100, w: 60, h: 30 } }], screenshot_width: args.max_image_dimension === 4096 ? 1000 : 500, screenshot_height: args.max_image_dimension === 4096 ? 700 : 350, window_bounds: bounds } };if (name === "zoom") return { content: [jpeg], structuredContent: { width: 200, height: 100 } };if (name === "get_desktop_state") return { content: [args.max_image_dimension === 0 ? raw : png], structuredContent: { display: "primary", platform: "windows", screenshot_width: args.max_image_dimension === 0 ? 1000 : 500, screenshot_height: args.max_image_dimension === 0 ? 700 : 350, screenshot_original_width: 1000, screenshot_original_height: 700, screen_width: 1000, screen_height: 700, scale_factor: 1 } };if (name === "click" && args.x !== undefined) return { content: [{ type: "text", text: "✅ Posted click to Chromium pid 42 at (231,241) (background, no foreground swap)." }], structuredContent: { path: "post_message", effect: "unverifiable" } };@@ -174,6 +174,37 @@ test("snapshot refinement previews the final annotated image and verification emexpect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "verify" } });expect(observer.mock.calls[0][0]).not.toHaveProperty("image");});+test.each(["after lifetime check", "during sample"] as const)("cancelled verification stays unknown %s and emits no completion", async phase => {+ const abort = new AbortController(); const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" }));+ const f = await fixture(observer); const ref = await outside(f.session);+ if (phase === "after lifetime check") {+ const identity = f.driver.identity;+ f.driver.identity = vi.fn<Driver["identity"]>(async (...args) => { const value = await identity(...args); abort.abort(); return value; });+ } else {+ const call = f.driver.call;+ f.driver.call = vi.fn<Driver["call"]>(async (...args) => { const value = await call(...args); if (args[0] === "get_window_state") abort.abort(); return value; });+ }+ const result = await f.session.waitFor(ref, f.session.waitSpec(ref, { action: "verify", conditions: [{ target: { role: "Button" }, is: "visible" }] }), abort.signal);+ expect(result.details).toMatchObject({ status: "observed", verification: { verdict: "unknown", samples: 0, timed_out: false } });+ expect(result.content.every(block => block.type !== "image")).toBe(true);+ expect(observer).not.toHaveBeenCalled();+ expect(vi.mocked(f.driver.call).mock.calls.filter(([name]) => name === "get_window_state")).toHaveLength(phase === "during sample" ? 1 : 0);+});++test.each(["cancel", "revoke"] as const)("verification completion rechecks %s without exposing a decided result", async change => {+ const abort = new AbortController(); const observer = vi.fn<WindowObserver>(async () => {+ if (change === "cancel") abort.abort(); else f.setMode("safe");+ return { status: "queued" };+ });+ const f = await fixture(observer); const ref = await outside(f.session);+ const result = await f.session.waitFor(ref, f.session.waitSpec(ref, { action: "verify", conditions: [{ target: { role: "Button" }, is: "visible" }] }), abort.signal);+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "verify" }, outcome: "verified" });+ if (change === "cancel") expect(result.details).toMatchObject({ status: "observed", verification: { verdict: "unknown", samples: 1 } });+ else { expect(result.details.status).toBe("refused"); expect(result.details).not.toHaveProperty("verification"); }+ expect(result.content.every(block => block.type !== "image")).toBe(true);+ expect(vi.mocked(f.driver.call).mock.calls.filter(([name]) => name === "get_window_state")).toHaveLength(1);+});test("a viewport previews its returned crop and marks only the proven physical region", async () => {const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" }));const f = await fixture(observer); const ref = await outside(f.session);